comparison cpack/dml/api/dmlvis.pl @ 0:718306e29690 tip

commiting public release

+/* Part of DML (Digital Music Laboratory)
+	Copyright 2014-2015 Samer Abdallah, University of London
+	 
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation; either version 2
+	of the License, or (at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public
+	License along with this library; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+:- module(dmlvis, 
+      [  cc/3
+      ,  recording_property/3 
+      ]).
+
+/** <module> DML Visualisation web service
+ */
+
+
+:- use_module(library(http/html_write)).
+:- use_module(library(http/html_head)).
+:- use_module(library(http/thread_httpd)).
+:- use_module(library(http/http_dispatch)).
+:- use_module(library(http/http_parameters)).
+:- use_module(library(http/http_json)).
+:- use_module(library(http/json)).
+:- use_module(library(thread_pool)).
+:- use_module(library(semweb/rdf_label)).
+:- use_module(library(sandbox)).
+:- use_module(library(dcg_core)).
+:- use_module(library(dcg_pair)).
+:- use_module(library(dcg_codes)).
+:- use_module(library(dcg_macros)).
+:- use_module(library(fileutils)).
+:- use_module(library(listutils)).
+:- use_module(library(optutils)).
+:- use_module(library(swipe)).
+:- use_module(library(memo)).
+:- use_module(library(rdfutils)).
+:- use_module(library(httpfiles)).
+:- use_module(library(dataset)).
+:- use_module(library(computations), [rows_cols/3]).
+:- use_module(library(csvutils), [uri_to_csv/2]).
+:- use_module(library(insist)).
+:- use_module(library(lambda)).
+:- use_module(library(dcg/basics), [string_without//2, integer//1]).
+:- use_module(library(solution_sequences)).
+
+:- use_module(components(audio)).
+
+:- meta_predicate cc(2,2,1).
+:- meta_predicate method_result_cc(+,+,2,1).
+
+:- set_prolog_flag(double_quotes,codes).
+:- set_prolog_flag(back_quotes,string).
+
+http:location(v1,root(api/v4),[]).
+
+:- http_handler(v1(getCollectionId), getCollectionId, []).
+:- http_handler(v1(getCollectionPerspective), getCollectionPerspective, []).
+:- http_handler(v1(getRecordingPerspective), getRecordingPerspective, [spawn(vis_recording)]).
+:- http_handler(v1(listCollections), listCollections, []).
+:- http_handler(v1(listPlaces), listPlaces, []).
+:- http_handler(v1(listPerspectives), listPerspectives, []).
+:- http_handler(v1(csv_time_window), csv_time_window, []).
+
+% ------------------------------------------------------------------------------------
+
+%% vis(+Method,+Request) is det.
+%
+%  General VIS API HTTP handler. Returns either JSON or JSONP depending on the
+%  format parameter. If JSONP is selected (the default), the callback parameter
+%  determines the Javascript callback function. The JSON result structure is of
+%  type
+%  ==
+%  { 
+%     query:  string ~'The original query, excluding protocol and host',
+%     result: (error_struct|A) ~'Result structure A or error information'
+%  }
+%  ==
+%  where 
+%  ==
+%  error_struct == { code: integer, decs:string }
+%  ==
+%  See individual methods for more information about their return types.
+vis(Method,Request) :-
+   get_time(T0),
+   method_params(Method,Params),
+   member(request_uri(Query),Request),
+   vis_parameters(Request, [format(Format), callback(Callback) | Params],ROpts),
+   (Format=jsonp -> Fmt=jsonp(Callback,ROpts); Fmt=json(ROpts)),
+   Responder=send_response(Fmt,Query,T0),
+   debug(dmlvis(method),'~q',[method_result_cc(Method,Params,Responder,writeln)]),
+   method_result_cc(Method,Params,dmlvis:Responder,error_cont(Query,Responder)).
+
+send_response(Fmt,Query,T0,Result,Status) :-
+   get_time(T1), DT=T1-T0,
+   debug(dmlvis(response),'Sending response after ~3f s to ~q',[DT,Query]),
+   once(reply_as(Fmt,Status,_{ query:Query, result: Result})).
+
+error_cont(Q,Cont,Ex) :-
+   debug(dmlvis(error),'Sending error ~W in response to ~q',[Ex,[quoted(true),max_depth(8)],Q]),
+   error_result(Ex,Result),
+   call(Cont,Result,error).
+
+vis_parameters(Request,Params,ReplyOpts) :-
+   Params1=[random(_, [optional(true)]) | Params],
+   http_parameters(Request, Params1, [attribute_declarations(param), form_data(AllParams)]),
+   findall(N, (member(N=_,AllParams), \+used_param(N,Params1)), Unused),
+   (select(indent,Unused,Unused1) -> ReplyOpts=[]; ReplyOpts=[width(0)], Unused1=Unused),
+   (select(dv,Unused1,Unused2) -> true; Unused2=Unused1),
+   insist(Unused2=[], unrecognised_parameters(Unused2)).
+
+used_param(N,ValidParams) :-
+   member(Q,ValidParams),
+   functor(Q,N,_).
+
+% result_response(Fmt,Q,Result,Status,Response) :- 
+%    with_output_to(string(Response),
+%       reply_as(Fmt,Status,_{ query:Q, result: Result})).
+
+reply_as(json(Opts),Status,Dict) :- !,
+   write_headers([Status,type(json)]),
+   json_write_dict(current_output, Dict, Opts).
+reply_as(jsonp(Cb,Opts),Status,Dict) :- !, 
+   write_headers([Status,type(jsonp)]),
+   write(Cb), write('('),
+   json_write_dict(current_output, Dict, Opts),
+   write(');').
+
+
+error_result(Ex, _{ errors:[Err] }) :- error_result1(Ex,Err).
+
+error_result1(dml_error(Code,Descriptor),_{ code: Code, desc:Descriptor }) :- !.
+
+error_result1(Ex, _{ code: 500, desc: Msg }) :-
+   message_to_string(Ex,Msg).
+
+% ------------------------------------------------------------------------------------
+% Handlers with documentation
+
+%% getCollectionId(+Request) is det.
+%
+%  Define a collection satisfying given search terms. Result is a collection
+%  ID, which may be the same as a previously defined collection if it consists
+%  of the same items. 
+%
+%  Parameters are as follows. In all cases, =|sclist(Type)|= denotes a semicolon
+%  separated list of values of given type OR the string '*', which is the default.
+%  The value '*' means that no filtering is done on the releveant property. The
+%  list is interpreted as an AND-list or an OR-list depending on which property 
+%  is being tested. String matching is case insensitive. 
+%  The only required parameter is =|dv|=. 
+%  ==
+%   dv         : integer              ~ database version
+%   library    : sclist(lib_name)     ~ OR-list of libraries to search
+%   collection : sclist(pattern)      ~ OR-list of BL collection name patterns
+%   composer   : sclist(string)       ~ OR-list of composer prefixes 
+%   performer  : sclist(string)       ~ OR-list of performer prefixes 
+%   title      : sclist(string)       ~ AND-list of title substrings 
+%   genre      : sclist(string)       ~ OR-list of genre substrings 
+%   place      : sclist(string)       ~ OR-list of place substrings 
+%   language   : sclist(string)       ~ OR-list of langauage name prefixes 
+%   year       : sclist(year) | range ~ OR-list of genre substrings 
+%   sample     : nonneg | 'all'       ~ sample a random subset of this size
+%   sv         : integer              ~ sample version for sample
+%
+%  lib_name --> 'bl' | 'charm' | 'ilm' | 'mazurka' | 'beets'.
+%  range    --> year, '-', year.
+%  ==
+%  NB. A =|pattern|= is a string which may contain '*' to match any sequence of 
+%  characters. The pattern must match the WHOLE value being tested, eg to search
+%  for the substring 'Botswana', you must include '*' at both ends of the pattern,
+%  eg =|'*botswana*'|=.
+%  
+%  A successful result is of type
+%  ==
+%  result(getCollectionId) == { 
+%     cid: string ~'collection id', 
+%     size: natural ~'number of items in collection' 
+%  }.
+%  ==
+getCollectionId(R) :- vis(getCollectionId,R).
+
+%% getCollectionPerspective(+Request) is det. 
+%
+%  Get a given perspective on a previously defined collection.
+%  Result will depend on the particular perspective chosen. 
+%  All perspectives require the parameter =|cid(string)|=. Most
+%  perspectives accept the parameters =|recompute(oneof([none,failed,force]))|=
+%  and =|vamp_on_demand(boolean)|=.
+%  Defined perspectives are
+%
+%  ==
+%  summary : [] -> { cid:string, size: natural, goal: string }.
+%
+%  list : 
+%     [ limit(natural)/5000, offset(natural)/0, sort_by(oneof([label, date]))/label ] 
+%  -> { cid:string, items: list( {uri:uri, label:string, audio:list(link(audio)), date:string }) }.
+%
+%  midi_pitch_histogram : 
+%     [ weighting(oneof([none,dur,vel]))/none ] 
+%  -> { values:list(integer), counts:list(natural) }.
+%
+%  pitch_histogram : 
+%     [ weighting(oneof([none,dur,vel]))/none, quant(natural)/5, min(integer)/20, max(integer)/100, lang(oneof([ml,r]))/ml ]
+%  -> { edges:list(nonneg), counts:list(natural) }.
+%
+%  tempo_histogram : 
+%     [ period(nonneg)/1, min(integer)/20, max(integer)/100, num_bins(natural)/50, lang(oneof([ml,r]))/ml ]
+%  -> { edges:list(nonneg), counts:list(natural) }.
+%
+%  mean_tempo_curve : 
+%     [ num_samples(natural)/20, lang(oneof([ml,r]))/ml ]
+%  -> { means:list(nonneg), std_devs:list(nonneg) }.
+%
+%  tonic_relative_pitch_class_histogram : [] 
+%  -> { values:list(string), counts:list(nonneg), ok_count:natural, failed:{prolog:list, python:list}}.
+%
+%  pitch_lookup :
+%     [ midi_pitch(between(0,127)), weighting(oneof([none,dur,vel]))/none,
+%       limit(natural)/5000, offset(natural)/0 ]
+%  -> { }.
+%
+%  tonic_histogram : [] -> { }.
+%
+%  pitch_class_histogram : [] -> { }.
+%
+%  tuning_stats : [] -> { }.
+%
+%  tuning_stats_by_year : [] -> { }.
+%
+%  places_hist : [] -> { }.
+%
+%  key_relative_chord_seq : 
+%     [  spm_minlen(natural)/2, spm_maxseqs(natural)/500, spm_algorithm(Alg)/'CM-SPADE',
+%        spm_ignore_n(natural)/1, spm_maxtime(number)/60, spm_minsupport(number)/50 ] 
+%  -> { }.
+%
+%  similarity :
+%     [  sim_downsample(number)/1,sim_clusters(number)/40,sim_reclimit(number)/2000,
+%        sim_type(string)/'euclidean',sim_features(string)/'chromagram',
+%        sim_compressor(string)/'zlib'] 
+%  -> { }.
+%  ==
+getCollectionPerspective(R) :- vis(getCollectionPerspective,R).
+
+%% getCollectionPairPerspective(+Request) is det.
+%
+%  Get a given comparative perspective on a pair of previously defined collections.
+%  Result will depend on the particular perspective chosen. Defined perspectives are:
+getCollectionPairPerspective(R) :- vis(getCollectionPairPerspective,R).
+
+%% getMultiCollectionPerspective(+Request) is det.
+%  Get a given comparative perspective on a set of previously defined collections.
+getMultiCollectionPerspective(R) :- vis(getMultiCollectionPerspective,R).
+
+%% getRecordingPerspective(+Request) is det.
+%  Get a given perspective for a given recording URI. Perspectives, their parameters and
+%  their output types are as follows (all perspectives require a uri parameter)
+%  ==
+%  properties  : [] -> { 
+%     library:string, 
+%     title:string, 
+%     composer:list(string), 
+%     performer:list(string),
+%     genre:list(string),
+%     place:string,
+%     language:string,
+%     recording_date:string
+%  }.
+%
+%  transcription      : [] -> { csv: uri(csv([time,dur,freq,vel,pitch_name])) }.
+%  transcription_fine : [] -> { csv: uri(csv([time,dur,freq,vel,pitch_name)) }.
+%  chords             : [] -> { csv: uri(csv([time,chord_name])) }.
+%  chords_notes       : [] -> { csv: uri(csv([time,dur,integer)) }.
+%  key                : [] -> { csv: uri(csv([time,integer,string])) }.
+%  key_tonic          : [] -> { csv: uri(csv) }.
+%  tempo              : [] -> { csv: uri(csv([time,nonneg,tempo_string])).
+%  beats              : [] -> { csv: uri(csv([time,tempo_string])) }.
+%  beatroot           : [] -> { csv: uri(csv([time])) }.
+%
+%  tempo_nonuniform : [] -> { times:list(float), values:list(float) }. 
+%  tempo_uniform    : [ lang(oneof([ml,r]))/ml ] -> { times:list(float), values:list(float) }. 
+%  tempo_normalised : [ lang(oneof([ml,r]))/ml ] -> { times:list(float), values:list(float) }. 
+%  tempo_histogram  : 
+%     [ period(nonneg)/1, num_bins(natural)/50, min(nonneg)/20, max(nonneg)/100, lang(oneof([ml,r]))/ml  ]
+%  -> { edges: list(float), counts: list(natural) }.
+%
+%  midi_pitch_histogram : 
+%     [ weighting(note_weight)/none ]
+%  -> { values: list(integer), counts: list(natural) }.
+%
+%  pitch_histogram : 
+%     [weighting(note_weight)/none, quant(natural)/5, min(integer)/20, max(integer)/100, lang(oneof([ml,r]))/ml ]
+%  -> { edges: list(float), counts: list(natural) }.
+%
+%  chord_histogram : [] -> { values:list(string), counts:list(natural) }.
+%
+%  spectrogram : [offset(nonneg)/0, length(nonneg)/60 ] -> { csv: uri }.
+%  ==
+%
+%  Supplementary types:
+%  ==
+%  note_weight --> 'none'; 'dur'; 'vel'; 'dur*vel'.
+%
+%  time == float.
+%  dur  == nonneg.
+%  freq == nonneg.
+%  vel  == nonneg.
+%  ==
+getRecordingPerspective(R) :- vis(getRecordingPerspective,R).
+
+%% listCollections(+Request) is det.
+%  Lists previously defined collections and the Prolog queries that define them.
+%  ==
+%  listCollections : [] -> { 
+%     collections: list({  cid       : string, 
+%                          query     : string, 
+%                          dv        : integer, 
+%                          size      : natural, 
+%                          timestamp : string 
+%                       })
+%  }.
+listCollections(R) :- vis(listCollections,R).
+
+%% listPlaces(+Request) is det.
+%  Lists known place names, currently the distinct values of the dcterms:spatial predicate.
+%  ==
+%  listPlaces : [] -> { 
+%     places: list({  name      : string 
+%                  })
+%  }.
+listPlaces(R) :- vis(listPlaces,R).
+
+%% listPerspectives(+Request) is det.
+%  Lists available perspective for a given method.
+listPerspectives(R) :- vis(listPerspectives,R).
+
+%% csv_time_window(+Request) is det.
+%
+%  Returns the contents of a CSV file between between given time limits,
+%  assuming the the first column is a time value.
+%  Also returns the time of the last row. Parameters and return structure are
+%  ==
+%  csv_time_window : 
+%     [uri(uri(csv)), start(float), end(float) ]
+%  -> { duration:float, columns: list(list) }.
+%  ==
+%  URI parameter must be the URI of a CSV file.
+%  Data is returned a list of lists, where each inner list is one column
+%  of the CSV.
+csv_time_window(R) :- vis(csv_time_window,R).
+
+
+% ------------------------------------------------------------------------------------
+% Implementation of methods
+
+:- multifile param/2.
+
+param( format,   [oneof([json,jsonp]), default(jsonp), description('Reply format')]).
+param( callback, [atom,default(jsonp_cb), description('Callback for jsonp reply')]).
+param( dv,       [nonneg, optional(false), description('Database version ID')]).
+
+param( cid,       [atom, optional(false), description('Collection ID')]).
+param( pid,       [atom, optional(false), description('Perspective ID')]).
+param( cids,      [atom, optional(false), description('Semicolon separated list of Collection IDs')]).
+param( uri,       [atom, optional(false), description('Item URI')]).
+param( library,   [atom, default(*), description('Semicolon separated list of libraries to search')]).
+param( genre,     [atom, default(*), description('Semicolon separated list of genre names, or "*"')]).
+param( year,      [atom, default(*), description('Semicolon separeted list of release years or a range (Y1-Y2) or "*"')]).
+param( composer,  [atom, default(*), description('Semicolon separated list of composers or "*"')]).
+param( performer, [atom, default(*), description('Semicolon separated list of performers or "*"')]).
+param( place,     [atom, default(*), description('Semicolon separated list of place names or "*"')]).
+param( language,  [atom, default(*), description('Semicolon separated list of language name prefixes or "*"')]).
+param( collection,[atom, default(*), description('Semicolon separated list of BL collection names or "*"')]).
+param( title,     [atom, default(*), description('Semicolon separated AND-list of title substrings or "*"')]).
+param( method,    [atom, optional(false), description('API method name')]).
+param( limit,     [integer, default(5000), description('Maximum number of things to return')]).
+param( offset,    [integer, default(0), description('Offset within list')]).
+param( sort_by,   [oneof([label,date]), default(label), description('Sort recordings list by this property')]).
+param( start,     [number, optional(false), description('Start of window in seconds')]).
+param( end,       [number, optional(false), description('End of window in seconds')]).
+param( sample,    [number, default(all), description('Sample a random subset of this size')]).
+param( sv,        [number, default(1), description('Version of random subset')]).
+param( midi_pitch,[nonneg, default(60), description('MIDI note number 0..127')]).
+
+method_params(listCollections,  []).
+method_params(listPlaces,  []).
+method_params(listPerspectives, [ method(_) ]).
+method_params(getCollectionId,  [ dv(_), library(_), genre(_), place(_), language(_), year(_), composer(_), 
+                                 performer(_), collection(_), title(_), sample(_), sv(_) ]).
+method_params(csv_time_window,  [ uri(_), start(_), end(_) ]).
+method_params(Method, [ pid(_) | Params ]) :-
+   once(perspective(Method,_)), 
+   setof(P, perspective_param_name(Method,P), Ps),
+   maplist(param_name_term,Ps,Params).
+
+param_name_term(Name,Term) :- functor(Term,Name,1).
+perspective_param_name(Method,Name) :-
+   perspective(Method,_,Specs,_),
+   member(S,Specs),
+   optspec_name(S,Name).
+
+   
+:- multifile perspective/4.
+
+%% cc(+Pred:pred(-A,-cc_status), +Cont:pred(+A,+cc_status), +ErrorCont:pred(+exception)) is det.
+%  Call a predicate with success and error continuations.
+%  Pred roduces a result of type A and a status. If it succeeds, the result and status are passed
+%  to the continuation Cont. Otherwise, an exception is passed to ErrorCont.
+cc(Goal,Cont,ErrorCont) :- catch(cc_wrapper(Goal,Cont), Ex, call(ErrorCont,Ex)).
+cc_wrapper(Goal,Cont) :- insist(call(Goal,Result,Status)), call(Cont,Result,Status).
+
+place_name(P) :- rdf(_,dcterms:spatial,literal(P)).
+
+%% method_result_cc(+Method,+Params:options,+C:success_cont,+E:error_cont) is det.
+%
+% Perspectives are handle using continuations to allow spawning
+% ==
+% success_cont == pred(+dict,+vis_status).
+% error_cont == pred(+exception).
+% vis_status ---> stable; unstable.
+% ==
+% Method must be a method id registered in perspective/2 or handled my method_resut/4.
+method_result_cc(Method,Params,Cont,ErrorCont) :-
+   perspective(Method,_), !,
+   option(pid(PId),Params),
+   catch((  insist( perspective(Method,PId,Specs,Pred), unknown_perspective(PId)),
+            insist( maplist(options_optspec(Params),Specs)),
+            call(Pred,Cont,ErrorCont)
+         ), Ex, call(ErrorCont,Ex)).
+
+% all non-perspective methods are handled in this thread
+method_result_cc(Method,Params,Cont,ErrorCont) :-
+   cc( method_result(Method,Params), Cont, ErrorCont).
+
+%% method_result(+Method, +Opts, -Result, -Status:vis_status) is det.
+%
+%  Handles methods =|listCollections, listPerspectives, getCollectionId, csv_time_window|=.
+method_result(listCollections, _, _{ collections:List }, unstable) :-
+   findall( _{ cid:Id, query:QA, dv:DV, size:SZ, timestamp:TS },
+            (  browse(dataset:dataset(dmlvis:Q, DV, Id, SZ, _),comp(_,Timestamp,_)-ok),
+               format_time(string(TS),'%FT%T%:z',Timestamp),
+               term_to_atom(Q,QA)
+            ),
+            List).
+
+method_result(listPlaces, _, _{ places:List }, unstable) :-
+   findall( _{ name:Name, count:N }, aggregate(count,place_name(Name),N), List).
+
+
+method_result(listPerspectives, Opts, _{ perspectives:List }, stable) :-
+   option(method(Method), Opts),
+   findall( P, perspective(Method,P), List).
+
+method_result(getCollectionId, Opts, _{ cid:Id, size:Size, full_size:FullSize }, stable) :-
+   build_dataset_query(Query,DBV,Size-FullSize,Opts,Remaining),
+   insist(Remaining=[], unrecognised_search_criteria(Remaining)),
+   insist(dataset_query_id( Query, DBV, Id),empty_dataset(Query)),
+   dataset_size(Id, Size).
+
+method_result(csv_time_window, Opts, _{ duration:Dur, columns:Columns }, stable) :-
+   maplist(options_optspec(Opts), [ \uri(URI), \start(Start), \end(End) ]),
+   uri_to_csv(URI,Rows),
+   insist(Rows\=[], empty_csv(URI)),
+
+   append(_,[EndRow],Rows), 
+   functor(EndRow,_,NumCols),
+   arg(1,EndRow,Dur),
+
+   numlist(1,NumCols,Is),
+   drop_while(row_before(Start),Rows,Rows1),
+   take_while(row_before(End),Rows1,Rows2),
+   rows_cols(Is,Rows2,Columns).
+
+row_before(T0,Row) :- arg(1,Row,T), T<T0.
+
+options_optspec(Opts,+O) :- option(O,Opts).
+options_optspec(Opts,O-Def) :- option(O,Opts,Def).
+% options_optspec(M,Opts,O>Goal) :- options_optspec(M,Opts,O), call(M:Goal).
+
+optspec_name(+O,Name) :- functor(O,Name,1).
+optspec_name(O-_,Name) :- functor(O,Name,1).
+% optspec_name(O>_,Name) :- optspec_name(O,Name).
+
+perspective(Method,Perspective) :- perspective(Method,Perspective,_,_).
+
+perspective(getRecordingPerspective,  properties, [+uri(URI)], cc(recording_info(URI))).
+perspective(getCollectionPerspective, summary,    [+cid(C)], cc(collection_summary(C))).
+perspective(getCollectionPerspective, list,       [+cid(C),limit(Lim)-5000,offset(Off)-0,sort_by(SortBy)-label], cc(collection_list(C,Lim,Off,SortBy))).
+
+collection_summary(Id,Result,stable) :-
+   insist(dataset_size(Id,Size), unknown_collection(Id)),
+   findall( _{goal:GoalA, dv:DV}, ( dataset_query_dv(Id,Goal,DV), 
+                                    term_to_atom(Goal,GoalA)     ), Queries),
+   Result = _{cid:Id, size:Size, queries:Queries }.
+
+
+collection_list(Id, Lim, Offset, SortBy, _{cid:Id, size:Size, items:Items}, stable) :-
+   insist(dataset_size(Id,Size), unknown_collection(Id)),
+   findall(SortVal-Item, dataset_itemx(SortBy,Id,Item,SortVal), KeyedItems), sort(KeyedItems,Sorted),
+   findall(Item,limit(Lim,offset(Offset,member(_-Item,Sorted))),Items).
+
+dataset_itemx(SortProp, Id, Itemx, SortVal) :-
+   dataset_item(Id,URI),
+   filter(Lib,URI,in_library),
+   findall(Prop:Val, (  member(Prop,[label,audio,date]),
+                        recording_property(URI,Lib,Prop,Val) ), Props),
+   (member(SortProp:Val,Props) -> SortVal=just(Val); SortVal=nothing),
+   dict_create(Itemx,_,[uri:URI | Props]).
+
+
+recording_info(URI, Result, stable) :-
+   insist(filter(Lib,URI,in_library),unknown_recording(URI)),
+   findall(Prop:Val, recording_property(URI,Lib,Prop,Val), Props),
+   dict_create(Result,_,[ library:Lib | Props ]).
+
+recording_property(URI,Prop,Val) :-
+   filter(Lib,URI,in_library),
+   recording_property(URI,Lib,Prop,Val).
+
+recording_property(URI,_,label,Label) :- 
+   rdf_display_label(URI,Label).
+recording_property(URI,_,audio,Links) :- 
+   % !!! HACK: force scraping for BL items here..
+   % DISABLED - seems to upset BL server...
+   % (rdf(URI,dml:blpage,_) -> once(bl_p2r:scrape_audio_link(URI,_)); true),
+   setof(L,F^audio_link(URI,L,F),Links).
+recording_property(URI,Lib,Prop,Val) :-
+   lib_property_read(Lib, Prop, Reader),
+   pred_values(URI,Lib:Prop,Reader,Val).
+
+
+pred_values(URI,Lib:Prop,-Converter,Val) :- 
+   lib_property(Lib,Prop,Pred),
+   once(rdf(URI,Pred,Obj)), 
+   call(Converter,Obj,Val).
+pred_values(URI,LibProp,+Converter,Vals) :-
+   findall(Val, pred_values(URI,LibProp,-Converter,Val), Vals).
+
+lib_property_read(_, collection, -literal_text).
+lib_property_read(_, composer,   +literal_text).
+lib_property_read(_, performer,  +literal_text).
+lib_property_read(_, title,      -literal_text).
+lib_property_read(_, date,       -literal_text).
+lib_property_read(_, place,      -literal_text).
+lib_property_read(_, language,   +literal_text).
+lib_property_read(ilm, genre,    +genre_label).
+
+genre_label(Genre,Label) :-
+   rdf(Genre,rdfs:label,Lit),
+   literal_text(Lit,Label).
+
+% ------------------- DEFINING NEW COLLECTIONS -----------------------
+
+build_dataset_query(Query,DBV,Size-FullSize) -->
+   select_option(dv(DBV)),
+   select_option(sample(Subset)),
+   select_option(sv(SV)),
+   full_query(FQ),
+   {  Subset=all -> Query=FQ, Size=FullSize
+   ;  Query=sample(Subset,SV,FQ),
+      aggregate_all(count,call(FQ,_),FullSize)
+   }.
+
+full_query(qc(Filters)) -->
+   run_left(seqmap(process_qopt, 
+                   [  qo( library,    atoms(LS), libraries(LS))
+                   ,  qo( collection, atoms(CC), any(collection,CC))
+                   ,  qo( title,      atoms(TS), all(title,TS))
+                   ,  qo( composer,   atoms(CS), any(composer,CS))
+                   ,  qo( performer,  atoms(PS), any(performer,PS))
+                   ,  qo( genre,      atoms(GS), any(genre,GS))
+                   ,  qo( year,       years(YS), year(YS))
+                   ,  qo( place,      atoms(PS), any(place,PS))
+                   ,  qo( language,   atoms(LNS), any(language,LNS))
+                   ]), Filters, []). 
+
+process_qopt(qo(OptName,Parser,Filter)) -->
+   { Opt=..[OptName,OptVal] }, 
+   \> option_default_select(Opt,'*'),
+   (  {OptVal='*'} -> []
+   ;  {parse_atom(Parser,OptVal)}, 
+      \< [Filter]
+   ).
+            
+sample(Size,_,Query,X) :-
+   setof(X, call(Query,X), All),
+   length(All,Total),
+   (  Total=<Size -> Subset=All
+   ;  randset(Size, Total, All, [], Subset)
+   ),
+   member(X,Subset).  
+
+randset(0, _, _) --> !.
+randset(K, N, [A|As]) -->
+   ({random(N)>=K} -> {K1=K}; {K1 is K-1}, cons(A)),
+   {N1 is N-1}, randset(K1,N1,As).
+
+
+qc(Filters,X) :- maplist(filter(Lib,X),Filters), filter(Lib,X,in_library).
+
+%% lib_property(-Library, -Property, -Predicate) is nondet.
+%
+%  This predicate manages the mapping from API search fields to RDF
+%  predicates. There are still some questions about:
+%
+%  $ composer : marcrel cmp, arr, lyr
+%  $ performer : marcrel prf, sng, cnd, drt
+:- rdf_meta lib_property(?,+,r).
+lib_property(bl,    collection, dcterms:isPartOf).
+lib_property(bl,    language, dcterms:language).
+lib_property(bl,    language, dc:language).
+lib_property(bl,    place, dcterms:spatial).
+lib_property(ilm,   genre, mo:genre).
+
+% lib_property(beets, date, beets:original_year).
+lib_property(ilm,   date, ilm:release_date).
+lib_property(charm, date, charm:recording_date).
+lib_property(mazurka, date, mazurka:recording_date).
+lib_property(bl,    date, dcterms:created).
+
+lib_property(charm, composer, charm:composer).
+lib_property(mazurka, composer, mazurka:composer).
+lib_property(bl,    composer, marcrel:cmp).
+lib_property(bl,    composer, marcrel:arr).
+lib_property(beets, composer, beets:composer).
+
+lib_property(charm, performer, charm:performer).
+lib_property(mazurka, performer, mazurka:performer).
+lib_property(bl,    performer, marcrel:prf).
+lib_property(bl,    performer, marcrel:sng).
+lib_property(beets, performer, beets:artist).
+lib_property(ilm,   performer, ilm:arist).
+
+lib_property(charm, title, charm:title).
+lib_property(mazurka, title, mazurka:title).
+lib_property(bl,    title, dc:title).
+lib_property(beets, title, beets:title).
+lib_property(ilm,   title, dc:title).
+
+lib_property_search(_,collection,X,   substring(X)).
+lib_property_search(_,place,     X,   substring(X)).
+lib_property_search(_,language,  X,   prefix(X)).
+lib_property_search(_,composer,  X,   prefix(X)).
+lib_property_search(_,performer, X,   prefix(X)).
+lib_property_search(_,title,     X,   substring(X)).
+lib_property_search(beets, date,  L-U, between(L,U1)) :- succ(U,U1). 
+lib_property_search(_Lib,   date,  L-U, between(LA,U1A)) :-
+   % Lib\=beets,
+   succ(U,U1), 
+   atom_number(LA,L), 
+   atom_number(U1A,U1). 
+
+%% filter(+Lib, -Resource, +SearchSpec) is nondet.
+filter(Lib,   _, libraries(Ls))  :- member(Lib,Ls).
+filter(Lib,   X, any(Prop,Vals)) :- member(Val,Vals), filter(Lib,X,Prop,Val).
+filter(Lib,   X, all(Prop,Vals)) :- maplist(filter(Lib,X,Prop),Vals).
+filter(Lib,   X, year(any(Ys)))  :- member(Y,Ys), filter(Lib,X,date,Y-Y).
+filter(Lib,   X, year(L-U))      :- filter(Lib,X,date,L-U).
+filter(Lib,   X, Prop-Val)       :- filter(Lib,X,Prop,Val).
+
+filter(charm, X, in_library)     :- rdf(X,charm:file_name,_,charm_p2r).
+filter(mazurka, X, in_library)   :- rdf(X,mazurka:pid,_,mazurka_p2r).
+filter(bl,    X, in_library)     :- rdf(X,rdf:type,mo:'Signal',bl_p2r).
+filter(ilm,   X, in_library)     :- rdf(X,mo:track_number,_,ilm_p2r).
+filter(beets, X, in_library)     :- rdf(X,rdf:type,mo:'AudioFile',beets_p2r).
+
+%% filter(+Lib, -Resource, +Property, +Value) is nondet.
+
+% filter(beets, X,genre,G) :- rdf_has(X,beets:genre,literal(substring(G),_)).
+filter(ilm,   X,genre,G) :- 
+   rdf(GR,rdfs:label,literal(substring(G),_),ilm_p2r),
+   rdf(GR,rdf:type,mo:'Genre',ilm_p2r),
+   rdf(X,mo:genre,GR).
+
+filter(Lib, X, Prop, Val) :- 
+   lib_property(Lib,Prop,Pred),
+   lib_property_search(Lib,Prop,Val,Search),
+   rdf(X,Pred,literal(Search,_)).
+
+% --------- parsers -----------
+
+% cids(Ids) --> seqmap_with_sep(",",alphanum,Ids).
+cids(Ids) --> semicolon_sep(atom_codes,Ids).
+
+% atoms('*') --> "*", !.
+atoms(AS) --> semicolon_sep(atom_codes,AS1), {maplist(downcase_atom,AS1,AS2),sort(AS2,AS)}.
+whole(A) --> string_without("",Codes), {atom_codes(A1,Codes), downcase_atom(A1,A)}.
+
+% years('*') --> "*", !.
+years(L-U) --> integer(L), "-", integer(U). 
+years(any(Ys)) --> semicolon_sep(number_codes,Ys1), {sort(Ys1,Ys)}.
+
+% alphanum(X) --> string_without(",",S), {atom_string(X,S)}.
+
+item(Conv,Item) --> string_without(";",Codes), {call(Conv,Item,Codes)}.
+semicolon_sep(Conv,Items) -->
+   seqmap_with_sep(";",item(Conv),Items).
+
+
+parse_atom(Phrase,Atom) :- 
+   atom_codes(Atom,Codes), 
+   insist( phrase(Phrase,Codes), parse_failure(Phrase)).
+
+thread_pool:create_pool(vis_recording) :-
+   current_prolog_flag(cpu_count,N),
+   thread_pool_create(vis_recording, N, [backlog(50)]).