comparison cpack/dml/lib/crawler.pl @ 0:718306e29690 tip

commiting public release

+/* Part of DML (Digital Music Laboratory)
+	Copyright 2014-2015 Samer Abdallah, University of London
+	 
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License
+	as published by the Free Software Foundation; either version 2
+	of the License, or (at your option) any later version.
+
+	This program is distributed in the hope that it will be useful,
+	but WITHOUT ANY WARRANTY; without even the implied warranty of
+	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+	GNU General Public License for more details.
+
+	You should have received a copy of the GNU General Public
+	License along with this library; if not, write to the Free Software
+	Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
+*/
+
+:- module(crawler, 
+      [ crawl_loaded/2
+      , crawl_ui//2
+      , crawl/2
+      , crawl/1
+      , crawl_reload_graph/1
+      ]).
+
+:- multifile authority/3.
+:- multifile source/3.
+
+:- use_module(library(settings)).
+:- use_module(library(http/http_dispatch)).
+:- use_module(library(http/http_parameters)).
+:- use_module(library(http/http_wrapper)).
+:- use_module(library(http/html_write)).
+:- use_module(library(semweb/rdf_db)).
+:- use_module(library(prov_schema)).
+:- use_module(library(httpfiles)).
+:- use_module(library(htmlutils), [element//2]).
+:- use_module(library(dcg_core)).
+:- use_module(library(insist)).
+:- use_module(components(basics)).
+:- use_module(components(messages)).
+:- use_module(user(user_db)).
+
+:- setting(enable_auto_crawl, boolean, true, "Enables automatic semantic web searches for recognised URIs").
+
+:- http_handler(root(crawl), crawl_handler, []).
+
+no_cache -->
+   html_post(head,meta(['http-equiv'='Cache-Control', content='no-cache, no-store, must-revalidate, max-age=0'],[])),
+   html_post(head,meta(['http-equiv'='Pragma', content='no-cache'],[])),
+   html_post(head,meta(['http-equiv'='Expires', content=0],[])).
+
+refresh(URL,Delay) -->
+   no_cache,
+   html_post(head,meta(['http-equiv'=refresh, content="~d;~w"-[Delay,URL]],[])).
+
+%% crawl_handler(+Request) is det.
+% web service to query given authority about given URI
+crawl_handler(Request) :-
+   authorized(write(default, load(lod))),
+	http_parameters(Request,
+		[	uri(URI, [optional(false), description("URI to search for")])
+		,	src(Src, [optional(false), description("Source ID")])
+      ,  return_to(Return, [ optional(true), description('URI to return to') ])
+      ,  return_after(Delay, [ default(2) ])
+      ,  messages(Msgs, [boolean, default(true)])
+		]),
+   debug(crawler,"Got request to consult ~w on ~w",[Src,URI]),
+   return_options(Return, Delay, Options),
+   (  Msgs=true
+   -> call_showing_messages(insist(crawl(URI,Src)),Options)
+   ;  call_without_messages(insist(crawl(URI,Src)),Options)
+   ).
+
+call_without_messages(Goal,Options) :-
+   catch( (Goal, Msg='Success'), Ex, (Msg='Error', print_message(error,Ex))),
+   (  option(return_to(Return),Options)
+   -> option(return_after(Delay),Options,2),
+      reply_html_page(cliopatria(default),[], [ h2(Msg), \refresh(Return,Delay) ], [unstable])
+   ;  reply_html_page(cliopatria(default),[], [ h2(Msg) ], [unstable])
+   ).
+
+return_options(Return, _, []) :- var(Return), !.
+return_options(Return, Delay, [ return_to(Return), return_after(Delay) ]).
+
+
+%% crawl_ui(+URI:resource,+NT:natural) is det.
+%  Component to allow user to trigger a crawl on a given URI, depending on
+%  NT, the number of triples which currently have it has subject.
+crawl_ui(URI,NT) -->
+   (  {setof(S-Auto, uri_authority(URI,S,Auto),Sources)}
+   -> (  {setof(S-A, (member(S-A,Sources), \+crawl_loaded(URI,S)), Untapped)}
+         % !!! should check permissions here
+      -> { http_current_request(Request), 
+           memberchk(request_uri(Here), Request) 
+         },
+         {debug(crawler,'Untapped sources: ~q',[Untapped])},
+         no_cache,
+         (  {setting(enable_auto_crawl,true)},
+            {setof(S, member(S-true,Untapped), AutoSources)}
+         -> html([ 'The following sources have been consulted automatically: ',
+                   \seqmap_with_sep(html(', '),element(code),AutoSources), '.', br([])
+                 ]),
+            {  debug(crawler,'consulting in parallel: ~w...',[AutoSources]),
+               concurrent_maplist(consult_source(URI),AutoSources,Statuses),
+               debug(crawler,'finished consulting on ~w.',[URI])
+            },
+            (  {member(ok,Statuses)}
+            -> refresh(Here,0),
+               html([b('At least one consultation succeeded; refreshing automatically.'),br([])])
+            ;  html([b('All consultations failed.'),br([])])
+            )
+         ;  html(p([ 'Click to consult one of the following sources for more information.'
+                   , br([]), \seqmap(consult_form(URI,Here),Untapped)
+                   ]))
+         )
+      ; []
+      )
+   ;  {NT>0} -> []
+   ;  html(p('No triples and no authorities known for this URI.'))
+   ).
+
+consult_source(URI,Source,Status) :-
+   catch( (crawl(URI,Source),       Status=ok), Ex,
+          (print_message(error,Ex), Status=error(Ex))).
+
+consult_form(URI,Here,Source-_) -->
+   {http_link_to_id(crawl_handler, [], FetchURL)},
+   {source_name(Source,Name)},
+   {source(Source,_,Opts), option(messages(Msgs),Opts,true)},
+   html(form([style="display:inline-block", action(FetchURL)],
+      [ \hidden(uri, URI),
+        \hidden(src, Source),
+        \hidden(return_to, Here),
+        \hidden(messages, Msgs),
+        input([ type(submit), value(Name) ])
+      ])).
+
+%% crawl_loaded(+URI:resource,+Source:atom) is semidet.
+%  True when URI has already been crawled and added to the RDF database.
+crawl_loaded(URI,Source) :-
+   source_uri_graph(Source,URI,Graph),
+   debug(crawler,'Checking if loaded: ~w in graph ~w from ~w',[URI,Graph,Source]),
+	rdf(Graph,prov:wasDerivedFrom,URI,Graph).
+	
+
+%% crawl_reload_graph(+Graph:atom) is det.
+%  Attempts to delete the named graph and reload it from all the URIs
+%  that it was derived from.
+crawl_reload_graph(Graph) :-
+   findall(URI,rdf(Graph,prov:wasDerivedFrom,URI,Graph),URIs),
+   length(URIs,NURIs),
+   print_message(information,crawl_reload(Graph,NURIs)),
+   rdf_transaction((
+      rdf_unload_graph(Graph),
+      maplist(reload_into(Graph),URIs)
+   )).
+
+reload_into(Graph,URI) :-
+   uri_authority(URI,Source,_),
+   source_uri_graph(Source,URI,Graph),
+   load_into(Graph,Source,URI).
+
+%% crawl(+URI:resource) is det.
+%  Looks for information about URI on all authorities claiming authority on it.
+%  Queries are made in parallel using concurrent_maplist/2.
+crawl(URI) :-
+   findall(S,uri_authority(URI,S,_),Sources),
+   concurrent_maplist(consult_source(URI),Sources,_).
+   
+
+%% crawl(+URI:resource,+Auth:atom) is det.
+%  Looks for information about URI on specified authority.
+crawl(URI,Source) :-
+	debug(crawler,"Consulting source ~w on ~w...",[Source,URI]),
+	source_uri_graph(Source, URI, Graph),
+   rdf_transaction(load_into(Graph,Source,URI)).
+
+load_into(Graph,Source,URI) :-
+   source(Source, Handler, _),
+   (  call(Handler,crawl(URI,Graph)) 
+   -> print_message(information,crawler_crawled(Source,URI,Graph))
+   ;  setof(T, call(Handler,describe(URI,T)), Triples),
+      length(Triples, NumTriples),
+      forall(member(rdf(S,P,O),Triples), rdf_assert(S,P,O,Graph))
+   -> print_message(information,crawler_described(Source,URI,Graph,NumTriples))
+   ;  print_message(information,crawler_failed(Source,URI))
+   ),
+   debug(crawler,'Asserting successful load of ~w into graph ~w from ~w',[URI,Graph,Source]),
+   rdf_assert(Graph, prov:wasDerivedFrom, URI, Graph).
+
+%% uri_authority(+URI:resource, -Auth:atom, -Auto:boolean) is nondet.
+%  True when Auth is declared as an authority on URI. If Auto=true,
+%  then, this authority should be queried automatically when URI is the 
+%  subject of a 'list_resource' page.
+uri_authority(URI,Src,Auto) :-
+   authority(Cond,Src,Opts),
+   once(matches(Cond,URI)),
+   option(auto(Auto),Opts,false).
+
+matches(begins(Prefix),URI) :- sub_atom(URI,0,_,_,Prefix).
+matches(\+Cond,URI) :- \+matches(Cond,URI).
+matches((C1,C2),URI) :- matches(C1,URI), matches(C2,URI).
+matches((C1;C2),URI) :- matches(C1,URI); matches(C2,URI).
+
+
+%% source_uri_graph(+Auth:atom,+URI:resource,-Graph:atom) is det.
+%  Derive the RDF graph name for a given authority.
+source_uri_graph(Source,URI,Graph) :-
+   source(Source,Handler,_),
+   call(Handler,uri_graph(URI,Graph)).
+
+source_name(Source,Name) :- source(Source,Handler,_), call(Handler,name(Name)), !.
+source_name(Source,Source).
+
+prolog:message(crawler_crawled(Src,URI,Graph)) -->
+   ["Triples from ~w about ~w loaded into ~w via crawl method"-[Src,URI,Graph]].
+prolog:message(crawler_described(Src,URI,Graph,N)) -->
+   ["Triples from ~w about ~w loaded into ~w via describe method: ~d."-[Src,URI,Graph,N]].
+prolog:message(crawler_failed(Src,URI)) -->
+   ["Failed to load anything from ~w about ~w."-[Src,URI]].
+