comparison cpack/dml/lib/vamp.pl @ 0:718306e29690 tip

commiting public release

+/* Part of DML (Digital Music Laboratory)
+	Copyright 2014-2015 Samer Abdallah, University of London
+	 
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation; either version 2
+	of the License, or (at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public
+	License along with this library; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+:- module(vamp, []). 
+
+:- use_module(library(semweb/rdf_db)).
+:- use_module(library(semweb/rdf_label)).
+:- use_module(library(archive)).
+:- use_module(library(settings)).
+:- use_module(library(fileutils), except([file_under/4])).
+:- use_module(library(termutils)).
+:- use_module(library(rdfutils)).
+:- use_module(library(xmlarchive)).
+:- use_module(library(dcg_core)).
+:- use_module(library(dcg_pair)).
+:- use_module(library(dcg/basics)).
+:- use_module(library(insist)).
+:- use_module(library(lambda)).
+:- use_module(entailment(p2r)).
+:- use_module(api(archive)).
+:- use_module(cliopatria(hooks)).
+
+:- set_prolog_flag(double_quotes,string).
+:- rdf_register_prefix(vamp,'http://purl.org/ontology/vamp/').
+:- rdf_register_prefix(vamp_plugins,'http://vamp-plugins.org/rdf/plugins/').
+
+
+% -------------- LABEL HOOKS FOR COMPUTATIONS AND TRANSFORMS ----------------
+
+rdf_label:display_label_hook(R,_,Label) :-
+   rdf(R,rdf:type,dml:'Computation'), !,
+   rdf(R,dml:'comp/function',Fn), rdf_display_label(Fn,FnLabel),
+   rdf(R,dml:'comp/input',Input), 
+   (  Input=literal(InputLabel) -> true
+   ;  rdf_display_label(Input,InputLabel)
+   ),
+   format(string(Label),'~s \u25B8 ~s',[InputLabel,FnLabel]).
+
+rdf_label:display_label_hook(R,_,Label) :- rdf(_,vamp:parameter,R), !, label(parameter,R,Label).
+% rdf_label:display_label_hook(R,_,Label) :- rdf(R,rdf:type,vamp:'Transform'), !, label(transform,R,Label).
+rdf_label:display_label_hook(R,_,Label) :- rdf(_,vamp:parameter_binding,R), !, label(binding,R,Label).
+
+label(binding,URI,Label) :-
+   rdf(URI,vamp:parameter,Param), label(parameter,Param,ParamLabel),
+   rdf(URI,vamp:value,Value),     rdf_literal_value(Value,Val),
+   format(string(Label),'~s=~w',[ParamLabel,Val]).
+
+
+label(parameter,URI,Label) :-
+   rdf(URI,vamp:identifier,ID), literal_text(ID,IDText),
+   (  rdf(URI,vamp:unit,Unit), 
+      literal_text(Unit,UnitText), 
+      UnitText\=''
+   -> literal_text(Unit,UnitText), 
+      format(string(Label),'~s (~s)',[IDText,UnitText])
+   ;  format(string(Label),'~s',[IDText])
+   ).
+
+label(transform,URI,Label) :-
+   rdf(URI,vamp:plugin,Plugin), rdf_display_label(Plugin,PluginName),
+   rdf(URI,vamp:output,Output), rdf_display_label(Output,OutputLabel),
+   (  setof(B,rdf(URI,vamp:parameter_binding,B),Bindings)
+   -> maplist(label(binding),Bindings,BindingLabels),
+      atomics_to_string(BindingLabels,', ',BindingsText),
+      format(string(Label),'~s (~s) | ~s',[PluginName,BindingsText,OutputLabel])
+   ;  format(string(Label),'~s | ~s',[PluginName,OutputLabel])
+      %cdot is \u22C5
+   ).
+
+% ---------------------- HIGH LEVEL OPS ---------------------------
+
+% :- rdf_meta make_triple(r,r,o,-).
+% make_triple(S,P,O,rdf(S,P,O)).
+
+:- public import/0.
+import :- 
+   installed_plugins(Plugins),
+   load_plugins(Plugins),
+   assert_all_to( vamp, plugins_triple(Plugins)),
+   assert_all_to( vamp, vamp_triple(transforms)),
+   assert_all_to( vamp, vamp_triple('_Audio_Analysis')),
+   assert_all_to( vamp, vamp_triple('_ILM_Analysis')).
+
+import_directory_graph(Dir,Graph) :-
+   must_be(ground,Graph),
+   must_be(atom,Dir),
+   retractall(failed(_,_)),
+   rdf_retractall(_,_,_,Graph),
+   with_status_line(assert_all_to(Graph, vamp_triple(Dir))).
+
+vamp_triple(Pattern,Triple) :-
+   absolute_file_name(archive(vamp),VampRoot),
+   atomic_list_concat([VampRoot,'/',Pattern],AbsPattern),
+   absolute_file_name(AbsPattern,AbsDir,[expand(true),file_type(directory)]),
+   atom_concat(VampRoot,RelDir,AbsDir),
+   atomic_list_concat([_|RelParts],'/',RelDir),
+   RelParts=[Top|_],
+   directory_triple(Top,[vamp|RelParts],AbsDir,Triple).
+
+
+installed_plugins(Plugins) :-
+   absolute_file_name(path('sonic-annotator'),_,[access(execute)]),
+   with_input_from_file(pipe('sonic-annotator -l'), read_lines_to_strings(current_input,Lines)),
+   setof(Library:Name, 
+         O^ID^(member(ID,Lines),atomic_list_concat([vamp,Library,Name,O],':',ID)), 
+         Plugins).
+
+load_plugins(Plugins) :-
+   setof(Library, Name^member(Library:Name, Plugins), Libraries),
+   maplist(\Lib^URL^rdf_global_id(vamp_plugins:Lib,URL), Libraries, URLs),
+   rdf_load(URLs,[]).
+
+:- rdf_meta plugins_triple(+,t).
+plugins_triple(Plugins,rdf(URI,rdf:type,dml:'Installed')) :-
+   rdf_current_prefix(vamp_plugins,PluginURLBase),
+   member(Library:Name,Plugins),
+   format(atom(URI),'~w~w#~w',[PluginURLBase,Library,Name]).
+
+
+directory_triple(transforms,DirParts,AbsDir,Triple) :- !,
+   append(DirParts,Parts,LocParts),
+   file_under(AbsDir,AbsPath,Parts),
+   file_name_extension(_, n3, AbsPath),
+   locator_uri( file(LocParts), URI),
+   dcg_triple( transform_triples(URI, with_file(AbsPath)), Triple).
+
+directory_triple('_ILM_Analysis',DirParts,AbsDir,Triple) :- !,
+   append(DirParts,Parts,LocParts),
+   file_under(AbsDir,_,Parts),
+   append(_,[FileName],Parts),
+   file_name_extension(BaseName,csv,FileName),
+   Locator=file(LocParts),
+   status("Importing: ~w",[Locator]),
+   string_codes(BaseName, BaseNameCodes),
+   (  phrase(ilm_filename(AlbumId,TrackNo,PluginAndOutput), BaseNameCodes)
+   -> true
+   ;  print_message(warning,failed_to_parse_ilm_file(LocParts)), fail
+   ),
+   insist(tmap(PluginAndOutput,TName)),
+   locator_uri(file([vamp,transforms,TName]),TransformGraph),
+   atom_concat(TransformGraph,'#transform',Transform),
+   locator_uri(Locator,FileURI),
+   pattern_uri(ilm:track/num(AlbumId)/num(TrackNo), AudioObject),
+   dcg_triple(computation_triples(_,AudioObject,Transform,FileURI), Triple).
+
+directory_triple(_,DirParts,AbsDir,Triple) :-
+   append(DirParts,Parts,LocParts),
+   file_under(AbsDir,AbsPath,Parts),
+   file_name_extension(_,Ext,AbsPath),
+   append([_,_|PathParts],[FileName],LocParts),
+   (  archive_ext(Ext)
+   -> fail % DISABLED
+      %append(PathParts,Tail,CompositeParts),
+      %atomics_to_string(LocParts,'/',RelPath),
+      %with_archive_file(AbsPath, Name, 
+      %   archive_entry_triple(CompositeParts-Tail,RelPath,Name,Triple))
+   ;  file_name_extension(BaseName,Ext,FileName),
+      dcg_triple( file_triples(Ext, BaseName, PathParts, file(LocParts),
+                               with_file(AbsPath)),
+                  Triple)
+   ).
+
+
+
+read_lines_to_strings(Stream,Lines) :-
+   read_line_to_string(Stream,String),
+   (  String=end_of_file
+   -> Lines=[]
+   ;  Lines=[String|Rest],
+      read_lines_to_strings(Stream,Rest)
+   ).
+
+ilm_filename(AlbumId,TrackNo,PluginAndOutput) -->
+   integer(AlbumId), "-",
+   integer(TrackNo), ".",
+   integer(_), "_vamp_",
+   string_without([],Codes),
+   {atom_codes(PluginAndOutput,Codes)}.
+
+tmap('beatroot-vamp_beatroot_beats',              'beatroot_standard.n3').
+tmap('qm-vamp-plugins_qm-chromagram_chromagram',  'qm-chromagram_standard.n3').
+tmap('qm-vamp-plugins_qm-mfcc_coefficients',      'qm-mfcc-standard.n3').
+tmap('qm-vamp-plugins_qm-keydetector_key',        'qm_vamp_key_standard.n3').
+tmap('qm-vamp-plugins_qm-keydetector_tonic',      'qm_vamp_key_standard_tonic.n3').
+tmap('qm-vamp-plugins_qm-segmenter_segmentation', 'qm-segmentation_standard.n3').
+tmap('qm-vamp-plugins_qm-tempotracker_beats',     'tempotracker_beats_standard.n3').
+tmap('qm-vamp-plugins_qm-tempotracker_tempo',     'tempotracker_tempo_standard.n3').
+
+
+with_file(File,Stream,Goal) :- 
+   with_stream(Stream,open(File,read,Stream),Goal).
+
+%% archive_entry(+Prefix:diff_list(string), +ArchivePath:string,+Name:atom,-Triple:triple,+Archive:archive) is nondet.
+%
+%  Produces all the triples associated with the current archive entry. ArchivePath is The relative 
+%  path to the archive from data root directory (in the setting vamp:data_directory).
+%  Prefix is a difference list containing the path components to be prefixed to the inter-archive
+%  path to generate the full path associated with the entry.
+archive_entry_triple(Parts-Tail,ArchivePath,Name,Triple,Archive) :- 
+   split_string(Name,"/","",InArchiveParts), 
+   append(Tail,[FileName],InArchiveParts), 
+   % Parts is now full composite path excluding file name
+   file_name_extension(BaseName,Ext,FileName),
+   dcg_triple( 
+      file_triples(Ext, BaseName, Parts, archive_entry(ArchivePath,Name), 
+                   with_current_entry_stream(Archive)),
+      Triple).
+
+archive_ext('7z').
+archive_ext(gz).
+archive_ext(bz2).
+archive_ext(zip).
+
+recompute_labels :-
+   forall( rdf(T,rdf:type,vamp:'Transform'), 
+      (  label(transform,T,LabelS),
+         atom_string(Label,LabelS),
+         (  rdf(T,rdfs:label,Old)
+         -> rdf_update(T,rdfs:label,Old,object(literal(Label)))
+         ;  rdf_assert(T,rdfs:label,literal(Label),vamp)
+         )
+      )
+   ).
+
+
+% ------------------- FILE IMPORTER, GENERAL PART ---------------------
+%
+
+% :- rdf_meta rdf(r,r,o,?,?).
+rdf(S,P,O) --> 
+   {  expand_resource(S,SS),
+      expand_resource(P,PP),
+      expand_resource(O,OO) 
+   },
+   [rdf(SS,PP,OO)].
+
+expand_resource(X,X) :- var(X), !, rdf_bnode(X).
+expand_resource(literal(X),O) :- !, rdf_global_object(literal(X),O).
+expand_resource(X,Y) :- pattern_uri(X,Y).
+
+computation_triples(Computation,Input,Function,Output) -->
+   (  {rdf(_,dml:'comp/output', Output,vamp)} -> [] % Already loaded
+   ;  rdf(Computation, rdf:type, dml:'Computation'),
+      rdf(Computation, dml:'comp/input',    Input),
+      rdf(Computation, dml:'comp/function', Function),
+      rdf(Computation, dml:'comp/output',   Output)
+   ).
+
+
+parse_dirname(Dirname,Hash) :-
+   sub_string(Dirname,_,_,After,".n3_"),
+   sub_string(Dirname,_,After,0,HashString),
+   atom_string(Hash,HashString).
+
+:- dynamic failed/2.
+:- meta_predicate file_triples(+,+,+,+,2,?,?).
+
+file_triples(n3,BaseName,_,_,Reader) --> !,
+   {  (sub_atom(BaseName,B,_,_,'_vamp_') -> B=<2; true), % to allow for qm_vampXXXXX.n3
+      sub_atom(BaseName,_,5,0,Hash), 
+      status("Importing transform: ~w.n3",[BaseName]),
+      pattern_uri(dml:transform/Hash,Graph) 
+   },
+   transform_triples(Graph,Reader).
+
+file_triples(csv,BaseName,PathParts,Locator,_) --> !,
+   {  status("Importing: ~w",[Locator]),
+      append(PathPrefix,[DirName],PathParts),
+      parse_dirname(DirName,Hash),
+      atomic_list_concat([transform,'/',Hash,'#',transform], Transform) ,
+      once(sub_string(BaseName,Bef,_,_,"_vamp")),
+      sub_string(BaseName,0,Bef,_,IDString),
+      atom_string(ID,IDString),
+      locator_uri(Locator,FileURI)
+   },
+   {  id_to_audio_uri(PathPrefix,ID,AudioObject) ->  true
+   ;  % print_message(warning,failed(id_to_audio_uri(PathParts,PathPrefix,DirName,ID,AudioObject))),
+      PathParts=[Collection|_], 
+      atomic_list_concat([Collection,ID],'/',AudioLocator),
+      humdrum_p2r:id_assert(vamp:failed(id_to_audio_uri(PathPrefix,ID,AudioObject),AudioLocator)),
+      AudioObject=literal(AudioLocator)
+   },
+   computation_triples(_, AudioObject, dml:Transform, FileURI).
+
+
+transform_triples(Graph,Reader) -->
+   {  rdf_graph(Graph) -> true % already loaded
+   ;  call(Reader,S,rdf_load(S,[format(turtle),silent(true),base_uri(Graph),graph(Graph)]))
+   },
+   { rdf(Transform,rdf:type,vamp:'Transform',Graph) },
+   (  { rdf(Transform,rdfs:label,_) } -> []
+   ;  { once(label(transform,Transform,LabelS)) }, % precompute label 
+      { atom_string(Label,LabelS) },
+      rdf(Transform,rdfs:label,literal(Label))
+   ).
+      
+:- use_module(library(memo)).
+:- volatile_memo bare_id_to_audio_uri(+atom,-maybe(atom)).
+bare_id_to_audio_uri(ID,just(URI)) :-
+   rdf(DigSig,bldata:path,literal(substring(ID),_)),
+   (  rdf(DigSig,mo:sampled_version_of,URI) -> true
+   ;  rdf(DigSig,rdf:type,mo:'Signal'), URI=DigSig
+   ).
+bare_id_to_audio_uri(_,nothing).
+
+%% id_to_audio_uri(+PathParts:list(atom), +ID:atom -URI:uri) is semidet.
+%
+% This predicate has to work out wich recording is being referred to
+% by the name of this output file. It is a not terribly reliable.
+id_to_audio_uri(PathParts,ID,URI) :-
+   (  PathParts = ['CHARM-Collection'|_]
+   -> rdf(URI,charm:file_name,literal(ID)) 
+   ;  PathParts = ['mazurka-dataset'|_]
+   -> atom_concat(pid,PID,ID),
+      rdf(URI,mazurka:pid,literal(PID))
+   ;  PathParts = ['_Non-music'|_]
+   -> bare_id_to_audio_uri(ID,URI)
+   ;  atomics_to_string(PathParts,"/",Dir),
+      atomic_list_concat([Dir,'/',ID,'.'],Prefix),
+      rdf(DigSig,bldata:path,literal(prefix(Prefix),_)),
+      (  rdf(DigSig,mo:sampled_version_of,URI) -> true
+      ;  rdf(DigSig,rdf:type,mo:'Signal'), URI=DigSig
+      )
+   ).
+
+
+%% missing_audio(-Path:atom,-Matches:list(uri)) is nondet.
+%
+%  This predicate help to find which recordings referred to by imported
+%  computations could not be found.
+missing_audio(Path,Matches) :-
+   setof(Path,SS^rdf(SS,dml:'comp/input',literal(Path)),Paths), 
+   member(Path,Paths), 
+   atomic_list_concat([_,Filename],'/',Path), 
+   sub_atom(Filename,0,_,2,II), 
+   (  setof(S-Lit,rdf(S,mo:available_as,literal(prefix(II),Lit)),M)
+   -> Matches=M
+   ;  Matches=[]
+   ).
+
+% show_counts(Name) -->
+%    \< get(D-F), 
+%    { status('directories: ~|~` t~d~3+, files: ~|~` t~d~5+, ~s',[D,F,Name]) }.
+
+
+assert_all_to(Graph,Pred) :- forall(call(Pred,rdf(S,P,O)), rdf_assert(S,P,O,Graph)).
+
+dcg_triple(Phrase,rdf(S,P,O)) :- 
+   call_dcg(Phrase,Triples,[]), 
+   member(rdf(S,P,O),Triples).
+
+pattern_uri(Pattern,URI) :- uripattern:pattern_uri(Pattern,URI), !.
+
+%% file_under(+Dir:path,-AbsPath:path,-Parts:list(string)) is nondet.
+%  Finds files under directory Dir, succeeding multiple times with AbsPath
+%  bound to the absolute path (as an atom), and Parts bound to a list
+%  of directory components ending with the file name.
+file_under(Spec,AbsPath,Parts) :-
+   absolute_file_name(Spec,Dir),
+   file_under(Dir,AbsPath,Parts,[]).
+
+%% file_under(+Root:path(dir), -File:path(file)) is nondet.
+file_under(Root,File) -->
+   {  atom_concat(Root,'/*',Pattern),
+      status("Expanding: ~w",[Pattern]),
+      expand_pattern(Pattern,Item)
+   },
+   (  {exists_file(Item)} 
+   -> {file_base_name(Item,Name), File=Item}, [Name]
+   ;  {exists_directory(Item)} 
+   -> {file_base_name(Item,DirName)}, [DirName],
+      file_under(Item,File)
+   ).
+
+graph_triples(G,Triples) :- findall(rdf(S,P,O),rdf(S,P,O,G),Triples).