Chris@0: /*  $Id$
Chris@0: 
Chris@0:     Part of SWI-Prolog
Chris@0: 
Chris@0:     Author:        Jan Wielemaker
Chris@0:     E-mail:        wielemak@science.uva.nl
Chris@0:     WWW:           http://www.swi-prolog.org
Chris@0:     Copyright (C): 2007, University of Amsterdam
Chris@0: 
Chris@0:     This program is free software; you can redistribute it and/or
Chris@0:     modify it under the terms of the GNU General Public License
Chris@0:     as published by the Free Software Foundation; either version 2
Chris@0:     of the License, or (at your option) any later version.
Chris@0: 
Chris@0:     This program is distributed in the hope that it will be useful,
Chris@0:     but WITHOUT ANY WARRANTY; without even the implied warranty of
Chris@0:     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
Chris@0:     GNU General Public License for more details.
Chris@0: 
Chris@0:     You should have received a copy of the GNU Lesser General Public
Chris@0:     License along with this library; if not, write to the Free Software
Chris@0:     Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
Chris@0: 
Chris@0:     As a special exception, if you link this library with other files,
Chris@0:     compiled with a Free Software compiler, to produce an executable, this
Chris@0:     library does not by itself cause the resulting executable to be covered
Chris@0:     by the GNU General Public License. This exception does not however
Chris@0:     invalidate any other reasons why the executable file might be covered by
Chris@0:     the GNU General Public License.
Chris@0: */
Chris@0: 
Chris@0: :- module(rdf_library,
Chris@0: 	  [ rdf_attach_library/1,	% +Dir
Chris@0: 	    rdf_load_library/1,		% +Ontology
Chris@0: 	    rdf_load_library/2,		% +Ontology, +Options
Chris@0: 	    rdf_list_library/0,
Chris@0: 	    rdf_list_library/1,		% +Ontology
Chris@0: 	    rdf_list_library/2,		% +Ontology, +Options
Chris@0: 	    rdf_library_index/2		% ?Id, ?Facet
Chris@0: 	  ]).
Chris@0: :- use_module(library('semweb/rdf_db')).
Chris@0: :- use_module(library('semweb/rdf_turtle')).
Chris@0: :- use_module(library(rdf)).
Chris@0: :- use_module(library(lists)).
Chris@0: :- use_module(library(option)).
Chris@0: :- use_module(library(debug)).
Chris@0: :- use_module(library(error)).
Chris@0: :- use_module(library(pairs)).
Chris@0: :- use_module(library(date)).
Chris@0: :- use_module(library(url)).
Chris@0: :- use_module(library(http/http_open)).
Chris@0: :- use_module(library(thread)).
Chris@0: 
Chris@0: /** <module> RDF Library Manager
Chris@0: 
Chris@0: This module manages an ontology library. Such   a  library consists of a
Chris@0: directory with manifest files  named   =manifest,rdf=  or =manifest.ttl=
Chris@0: (Turtle). The manifest files define ontologies  appearing in the library
Chris@0: as well as namespace mnemonics and dependencies.
Chris@0: 
Chris@0: The typical usage scenario is
Chris@0: 
Chris@0: ==
Chris@0: ?- rdf_attach_library('/some/directory').
Chris@0: ?- rdf_load_library(my_ontology).
Chris@0: ==
Chris@0: 
Chris@0: 
Chris@0: @tbd	Add caching info
Chris@0: @tbd	Allow HTTP-hosted repositories
Chris@0: 
Chris@0: @author Jan Wielemaker
Chris@0: */
Chris@0: 
Chris@0: :- rdf_register_ns(lib, 'http://www.swi-prolog.org/rdf/library/').
Chris@0: 
Chris@0: :- dynamic
Chris@0: 	manifest/2,			% Path, Time
Chris@0: 	library_db/3.			% Name, URL, Facets
Chris@0: 
Chris@0: %	Force compile-time namespace expansion
Chris@0: 
Chris@0: :- rdf_meta
Chris@0: 	edge(+, r,r,o).
Chris@0: 
Chris@0: 		 /*******************************
Chris@0: 		 *	      LOADING		*
Chris@0: 		 *******************************/
Chris@0: 
Chris@0: %%	rdf_load_library(+Id) is det.
Chris@0: %%	rdf_load_library(+Id, +Options) is det.
Chris@0: %
Chris@0: %	Load ontologies from the  library.  A   library  must  first  be
Chris@0: %	attached using rdf_attach_library/1.  Defined Options are:
Chris@0: %	
Chris@0: %		* import(Bool)
Chris@0: %		If =true= (default), also load ontologies that are
Chris@0: %		explicitely imported.
Chris@0: %		
Chris@0: %		* base_uri(URI)
Chris@0: %		BaseURI used for loading RDF.  Local definitions in
Chris@0: %		ontologies overrule this option.
Chris@0: %		
Chris@0: %		* claimed_source(URL)
Chris@0: %		URL from which we claim to have loaded the data.
Chris@0: %		
Chris@0: %		* not_found(+Level)
Chris@0: %		The system does a pre-check for the existence of
Chris@0: %		all references RDF databases.  If Level is =error=
Chris@0: %		it reports missing databases as an error and fails.
Chris@0: %		If =warning= it prints them, but continues.  If
Chris@0: %		=silent=, no checks are preformed.  Default is =error=.
Chris@0: %		
Chris@0: %		* concurrent(Threads)
Chris@0: %		Perform the load concurrently using N threads.  If not
Chris@0: %		specified, the number is determined by
Chris@0: %		guess_concurrency/2.
Chris@0: %	
Chris@0: %		* load(+Bool)
Chris@0: %		If =false=, to all the preparation, but do not execute
Chris@0: %		the actual loading.  See also rdf_list_library/2.
Chris@0: 
Chris@0: rdf_load_library(Id) :-
Chris@0: 	rdf_load_library(Id, []).
Chris@0: 
Chris@0: rdf_load_library(Id, Options) :-
Chris@0: 	load_commands(Id, Options, Pairs),
Chris@0: 	pairs_values(Pairs, Commands),
Chris@0: 	list_to_set(Commands, Cmds2),
Chris@0: 	delete_virtual(Cmds2, Cmds3),
Chris@0: 	find_conflicts(Cmds3),
Chris@0: 	check_existence(Cmds3, Cmds, Options),
Chris@0: 	(   option(concurrent(Threads), Options)
Chris@0: 	->  true
Chris@0: 	;   guess_concurrency(Cmds, Threads)
Chris@0: 	),
Chris@0: 	length(Cmds, NSources),
Chris@0: 	print_message(informational, rdf(loading(NSources, Threads))),
Chris@0: 	(   option(load(true), Options, true)
Chris@0: 	->  concurrent(Threads, Cmds,
Chris@0: 		       [ local(2000),	% we only need small stacks
Chris@0: 			 global(4000),
Chris@0: 			 trail(4000)
Chris@0: 		       ])
Chris@0: 	;   true
Chris@0: 	).
Chris@0: 
Chris@0: delete_virtual([], []).
Chris@0: delete_virtual([virtual(_)|T0], T) :- !,
Chris@0: 	delete_virtual(T0, T).
Chris@0: delete_virtual([H|T0], [H|T]) :-
Chris@0: 	delete_virtual(T0, T).
Chris@0: 
Chris@0: 
Chris@0: %%	find_conflicts(+LoadCommands) is semidet.
Chris@0: %
Chris@0: %	Find possibly conflicting options for loading the same source
Chris@0: 
Chris@0: find_conflicts(Commands) :-
Chris@0: 	sort(Commands, Cmds),
Chris@0: 	conflicts(Cmds, Conflicts),
Chris@0: 	report_conflics(Conflicts),
Chris@0: 	Conflicts == [].
Chris@0: 
Chris@0: conflicts([], []).
Chris@0: conflicts([C1, C2|T0], [C1-C2|T]) :-
Chris@0: 	conflict(C1, C2), !,
Chris@0: 	conflicts([C2|T0], T).
Chris@0: conflicts([_|T0], T) :-
Chris@0: 	conflicts(T0, T).
Chris@0: 
Chris@0: conflict(rdf_load(Src, Options1), rdf_load(Src, Options2)) :-
Chris@0: 	sort(Options1, S1),
Chris@0: 	sort(Options2, S2),
Chris@0: 	S1 \== S2.
Chris@0: 
Chris@0: report_conflics([]).
Chris@0: report_conflics([C1-C2|T]) :-
Chris@0: 	print_message(warning, rdf(load_conflict(C1,C2))),
Chris@0: 	report_conflics(T).
Chris@0: 
Chris@0: 
Chris@0: %%	check_existence(+CommandsIn, -Commands, +Options) is det.
Chris@0: %
Chris@0: %	Report existence errors. Fail if at   least  one source does not
Chris@0: %	exist. and the not_found level is not =silent=.
Chris@0: %	
Chris@0: %	@error existence_error(urls, ListOfUrls)
Chris@0: 
Chris@0: check_existence(CommandsIn, Commands, Options) :-
Chris@0: 	option(not_found(Level), Options, error),
Chris@0: 	must_be(oneof([error,warning,silent]), Level),
Chris@0: 	(   Level == silent
Chris@0: 	->  true
Chris@0: 	;   missing_urls(CommandsIn, Commands, Missing),
Chris@0: 	    (	Missing == []
Chris@0: 	    ->	true
Chris@0: 	    ;	Level == warning
Chris@0: 	    ->	report_missing(Missing, Level)
Chris@0: 	    ;	existence_error(urls, Missing)
Chris@0: 	    )
Chris@0: 	).
Chris@0: 
Chris@0: 
Chris@0: missing_urls([], [], []).
Chris@0: missing_urls([H|T0], Cmds, Missing) :-
Chris@0: 	H = rdf_load(URL, _),
Chris@0: 	(   exists_url(URL)
Chris@0: 	->  Cmds = [H|T],
Chris@0: 	    missing_urls(T0, T, Missing)
Chris@0: 	;   Missing = [URL|T],
Chris@0: 	    missing_urls(T0, Cmds, T)
Chris@0: 	).
Chris@0: 
Chris@0: report_missing([], _).
Chris@0: report_missing([H|T], Level) :-
Chris@0: 	print_message(Level, error(existence_error(url, H), _)),
Chris@0: 	report_missing(T, Level).
Chris@0: 
Chris@0: %%	guess_concurrency(+Commands, -Threads) is det.
Chris@0: %
Chris@0: %	How much concurrency to use? Set to   the  number of CPUs if all
Chris@0: %	input comes from  files  or  5   if  network  based  loading  is
Chris@0: %	demanded.
Chris@0: 
Chris@0: guess_concurrency(Commands, Threads) :-
Chris@0: 	count_non_file_url(Commands, Count),
Chris@0: 	(   current_prolog_flag(cpu_count, CPUs)
Chris@0: 	->  true
Chris@0: 	;   CPUs = 1
Chris@0: 	),
Chris@0: 	Threads is max(CPUs, min(5, Count)).
Chris@0: 
Chris@0: count_non_file_url([], 0).
Chris@0: count_non_file_url([rdf_load(URL, _)|T], Count) :-
Chris@0: 	sub_atom(URL, 0, _, _, 'file://'), !,
Chris@0: 	count_non_file_url(T, Count).
Chris@0: count_non_file_url([_|T], Count) :-
Chris@0: 	count_non_file_url(T, C0),
Chris@0: 	Count is C0 + 1.
Chris@0: 
Chris@0: 
Chris@0: %%	load_commands(+Id, +Options, -Pairs:list(Level-Command)) is det.
Chris@0: %
Chris@0: %	Commands are the RDF commands to execute for rdf_load_library/2.
Chris@0: %	Splitting  in  command  collection  and   execution  allows  for
Chris@0: %	concurrent execution as well  as   forward  checking of possible
Chris@0: %	problems.
Chris@0: %	
Chris@0: %	@tbd	Fix poor style; avoid assert/retract.
Chris@0: 
Chris@0: :- thread_local
Chris@0: 	command/2.
Chris@0: 
Chris@0: load_commands(Id, Options, Commands) :-
Chris@0: 	retractall(command(_,_)),
Chris@0: 	rdf_update_library_index,
Chris@0: 	dry_load(Id, 1, Options),
Chris@0: 	findall(Level-Cmd, retract(command(Level, Cmd)), Commands).
Chris@0: 
Chris@0: dry_load(Id, Level, Options) :-
Chris@0: 	(   library(Id, File, Facets)
Chris@0: 	->  merge_base_uri(Facets, Options, Options1),
Chris@0: 	    merge_source(Facets, Options1, Options2),
Chris@0: 	    merge_blanks(Facets, Options2, Options3),
Chris@0: 	    (   \+ memberchk(virtual, Facets)
Chris@0: 	    ->  load_options(Options3, File, RdfOptions),
Chris@0: 		assert(command(Level, rdf_load(File, RdfOptions)))
Chris@0: 	    ;	assert(command(Level, virtual(File)))
Chris@0: 	    ),
Chris@0: 	    (	option(import(true), Options, true)
Chris@0: 	    ->	Level1 is Level + 1,
Chris@0: 	        forall(member(imports(_, Import), Facets),
Chris@0: 		       import(Import, Level1, Options3))
Chris@0: 	    ;	true
Chris@0: 	    )
Chris@0: 	;   existence_error(ontology, Id)
Chris@0: 	).
Chris@0: 
Chris@0: merge_base_uri(Facets, Options0, Options) :-
Chris@0: 	(   option(base_uri(Base), Facets)
Chris@0: 	->  delete(Options0, base_uri(_), Options1),
Chris@0: 	    Options = [base_uri(Base)|Options1]
Chris@0: 	;   Options = Options0
Chris@0: 	).
Chris@0: 
Chris@0: merge_source(Facets, Options0, Options) :-
Chris@0: 	(   option(claimed_source(Base), Facets)
Chris@0: 	->  delete(Options0, claimed_source(_), Options1),
Chris@0: 	    Options = [claimed_source(Base)|Options1]
Chris@0: 	;   Options = Options0
Chris@0: 	).
Chris@0: 
Chris@0: merge_blanks(Facets, Options0, Options) :-
Chris@0: 	(   option(blank_nodes(Share), Facets)
Chris@0: 	->  delete(Options0, blank_nodes(_), Options1),
Chris@0: 	    Options = [blank_nodes(Share)|Options1]
Chris@0: 	;   Options = Options0
Chris@0: 	).
Chris@0: 
Chris@0: load_options(Options, File, RDFOptions) :-
Chris@0: 	findall(O, load_option(Options, File, O), RDFOptions).
Chris@0: 
Chris@0: load_option(Options, File, db(Source)) :-
Chris@0: 	option(claimed_source(Source0), Options),
Chris@0: 	(   sub_atom(Source0, _, _, 0, /)
Chris@0: 	->  file_base_name(File, Base),
Chris@0: 	    atom_concat(Source0, Base, Source)
Chris@0: 	;   atom_concat(Source, #, Source0)
Chris@0: 	->  true
Chris@0: 	).
Chris@0: load_option(Options, File, base_uri(BaseURI)) :-
Chris@0: 	option(base_uri(Base0), Options),
Chris@0: 	sub_atom(/, _, _, 0, Base0),
Chris@0: 	atom_concat(Base0, File, BaseURI).
Chris@0: load_option(Options, _File, blank_nodes(Share)) :-
Chris@0: 	option(blank_nodes(Share), Options).
Chris@0: 
Chris@0: %%	import(+URL, +Level, +Options) is det.
Chris@0: 
Chris@0: import(Path, Level, Options) :-
Chris@0: 	(   (   library(Id, Path, _)
Chris@0: 	    ->	true
Chris@0: 	    ;	manifest_for_path(Path, Manifest),
Chris@0: 		catch(exists_url(Manifest), _, fail)
Chris@0: 	    ->  process_manifest(Manifest),
Chris@0: 		library(Id, Path, _)
Chris@0: 	    )
Chris@0: 	->  dry_load(Id, Level, Options)
Chris@0: 	;   load_options(Options, Path, RdfOptions),
Chris@0: 	    assert(command(Level, rdf_load(Path, RdfOptions)))
Chris@0: 	).
Chris@0: 
Chris@0: manifest_for_path(URL, Manifest) :-
Chris@0: 	file_directory_name(URL, Parent),
Chris@0: 	manifest_file(Base),
Chris@0: 	rdf_extension(Ext),
Chris@0: 	concat_atom([Parent, /, Base, '.', Ext], Manifest).
Chris@0: 
Chris@0: %%	rdf_list_library(+Id) is det.
Chris@0: %%	rdf_list_library(+Id, +Options) is det.
Chris@0: %
Chris@0: %	Print library dependency tree to the terminal.  Options include
Chris@0: %	options for rdf_load_library/2 and
Chris@0: %	
Chris@0: %		* show_source(+Boolean)
Chris@0: %		If =true= (default), show location we are loading
Chris@0: %		
Chris@0: %		* show_graph(+Boolean)
Chris@0: %		If =true= (default =false=), show name of graph
Chris@0: %		
Chris@0: %		* show_virtual(+Boolean)
Chris@0: %		If =false= (default =true=), do not show virtual
Chris@0: %		repositories.
Chris@0: %		
Chris@0: %		* indent(Atom)
Chris@0: %		Atom repeated for indentation levels
Chris@0: 
Chris@0: rdf_list_library(Id) :-
Chris@0: 	rdf_list_library(Id, []).
Chris@0: rdf_list_library(Id, Options) :-
Chris@0: 	load_commands(Id, Options, Commands),
Chris@0: 	maplist(print_load(Options), Commands).
Chris@0: 
Chris@0: print_load(Options, _Level-virtual(_)) :-
Chris@0: 	option(show_virtual(false), Options), !.
Chris@0: print_load(Options, Level-Command) :-
Chris@0: 	option(indent(Indent), Options, '. '),
Chris@0: 	forall(between(2, Level, _), format(Indent)),
Chris@0: 	print_command(Command, Options),
Chris@0: 	format('~N').
Chris@0: 
Chris@0: print_command(virtual(URL), _Options) :-
Chris@0: 	format('<~w>', [URL]).
Chris@0: print_command(rdf_load(URL), Options) :-
Chris@0: 	print_command(rdf_load(URL, []), Options).
Chris@0: print_command(rdf_load(URL, RDFOptions), Options) :-
Chris@0: 	(   option(show_source(true), Options, true)
Chris@0: 	->  format('~w', [URL]),
Chris@0: 	    (   option(blank_nodes(noshare), RDFOptions)
Chris@0: 	    ->  format(' <not shared>')
Chris@0: 	    ;   true
Chris@0: 	    ),
Chris@0: 	    (   exists_url(URL)
Chris@0: 	    ->  true
Chris@0: 	    ;   format(' [NOT FOUND]')
Chris@0: 	    )
Chris@0: 	;   true
Chris@0: 	),
Chris@0: 	(   option(show_graph(true), Options, false),
Chris@0: 	    option(db(Base), RDFOptions)
Chris@0: 	->  format('~N\tSource: ~w', [Base])
Chris@0: 	;   true
Chris@0: 	).
Chris@0: 
Chris@0: exists_url(URL) :-
Chris@0: 	rdf_db:rdf_input(URL, Source, _BaseURI), 
Chris@0: 	exists_source(Source).
Chris@0: 
Chris@0: exists_source(file(Path)) :- !,
Chris@0: 	access_file(Path, read).
Chris@0: exists_source(url(http, URL)) :- !,
Chris@0: 	catch(http_open(URL, Stream, [ method(head) ]), _, fail),
Chris@0: 	close(Stream).
Chris@0: 	
Chris@0: 
Chris@0: %%	rdf_list_library
Chris@0: %
Chris@0: %	Prints known RDF library identifiers to current output.
Chris@0: 
Chris@0: rdf_list_library :-
Chris@0: 	rdf_update_library_index,
Chris@0: 	(   rdf_library_index(Id, title(Title)),
Chris@0: 	    format('~w ~t~20|~w', [Id, Title]),
Chris@0: 	    (	rdf_library_index(Id, version(Version))
Chris@0: 	    ->	format(' (version ~w)', [Version])
Chris@0: 	    ;	true
Chris@0: 	    ),
Chris@0: 	    nl,
Chris@0: 	    fail
Chris@0: 	;   true
Chris@0: 	).
Chris@0: 
Chris@0: 
Chris@0: %%	rdf_library_index(?Id, ?Facet) is nondet.
Chris@0: %
Chris@0: %	Query the content of the library.  Defined facets are:
Chris@0: %	
Chris@0: %		* source(URL)
Chris@0: %		Location from which to load the ontology
Chris@0: %		
Chris@0: %		* title(Atom)
Chris@0: %		Title used for the ontology
Chris@0: %		
Chris@0: %		* comment(Atom)
Chris@0: %		Additional comments for the ontology
Chris@0: %		
Chris@0: %		* version(Atom)
Chris@0: %		Version information on the ontology
Chris@0: %		
Chris@0: %		* imports(Type, URL)
Chris@0: %		URLs needed by this ontology. May succeed multiple
Chris@0: %		times.  Type is one of =ontology=, =schema= or =instances=.
Chris@0: %		
Chris@0: %		* base_uri(BaseURI)
Chris@0: %		Base URI to use when loading documents. If BaseURI
Chris@0: %		ends in =|/|=, the actual filename is attached.
Chris@0: %		
Chris@0: %		* claimed_source(Source)
Chris@0: %		URL from which we claim to have loaded the RDF. If
Chris@0: %		Source ends in =|/|=, the actual filename is
Chris@0: %		attached.
Chris@0: %		
Chris@0: %		* blank_nodes(Share)
Chris@0: %		Defines how equivalent blank nodes are handled, where
Chris@0: %		Share is one of =share= or =noshare=.  Default is to
Chris@0: %		share.
Chris@0: %		
Chris@0: %		* provides_ns(URL)
Chris@0: %		Ontology provides definitions in the namespace URL.
Chris@0: %		The formal definition of this is troublesome, but in
Chris@0: %		practice it means the ontology has triples whose
Chris@0: %		subjects are in the given namespace.
Chris@0: %		
Chris@0: %		* uses_ns(URL)
Chris@0: %		The ontology depends on the given namespace.  Normally
Chris@0: %		means it contains triples that have predicates or
Chris@0: %		objects in the given namespace.
Chris@0: %		
Chris@0: %		* manifest(Path)
Chris@0: %		Manifest file this ontology is defined in
Chris@0: %		
Chris@0: %		* virtual
Chris@0: %		Entry is virtual (cannot be loaded)
Chris@0: 
Chris@0: rdf_library_index(Id, Facet) :-
Chris@0: 	library(Id, Path, Facets),
Chris@0: 	(   Facet = source(Path)
Chris@0: 	;   member(Facet, Facets)
Chris@0: 	).
Chris@0: 
Chris@0: 
Chris@0: 		 /*******************************
Chris@0: 		 *	MANIFEST PROCESSING	*
Chris@0: 		 *******************************/
Chris@0: 
Chris@0: %%	rdf_attach_library(+Source)
Chris@0: %
Chris@0: %	Attach manifest from Source.  Source is one of
Chris@0: %	
Chris@0: %		* URL
Chris@0: %		Load single manifest from this URL
Chris@0: %		* File
Chris@0: %		Load single manifest from this file
Chris@0: %		* Directory
Chris@0: %		Scan all subdirectories and load all =|Manifest.ttl|= or
Chris@0: %		=|Manifest.rdf|= found.
Chris@0: %		
Chris@0: %	Encountered namespaces are registered   using rdf_register_ns/2.
Chris@0: %	Encountered ontologies are added to the index. If a manifest was
Chris@0: %	already loaded it will be reloaded  if the modification time has
Chris@0: %	changed.
Chris@0: 
Chris@0: rdf_attach_library(URL) :-
Chris@0: 	atom(URL),
Chris@0: 	is_absolute_url(URL), !,
Chris@0: 	process_manifest(URL).
Chris@0: rdf_attach_library(File) :-
Chris@0: 	absolute_file_name(File, Path,
Chris@0: 			   [ extensions([rdf,ttl]),
Chris@0: 			     access(read),
Chris@0: 			     file_errors(fail)
Chris@0: 			   ]), !,
Chris@0: 	process_manifest(Path).
Chris@0: rdf_attach_library(Dir) :-
Chris@0: 	absolute_file_name(Dir, Path,
Chris@0: 			   [ file_type(directory),
Chris@0: 			     access(read)
Chris@0: 			   ]),
Chris@0: 	attach_dir(Path, []).
Chris@0: 
Chris@0: 
Chris@0: %%	rdf_update_library_index
Chris@0: %
Chris@0: %	Reload all Manifest files.
Chris@0: 
Chris@0: rdf_update_library_index :-
Chris@0: 	forall(manifest(Location, _Time),
Chris@0: 	       process_manifest(Location)).
Chris@0: 
Chris@0: attach_dir(Path, Visited) :-
Chris@0: 	memberchk(Path, Visited), !.
Chris@0: attach_dir(Path, Visited) :-
Chris@0: 	atom_concat(Path, '/*', Pattern),
Chris@0: 	expand_file_name(Pattern, Members),
Chris@0: 	(   member(Manifest, Members),
Chris@0: 	    is_manifest_file(Manifest)
Chris@0: 	->  process_manifest(Manifest)
Chris@0: 	;   print_message(silent, rdf(no_manifest(Path)))
Chris@0: 	),
Chris@0: 	(   member(Dir, Members),
Chris@0: 	    exists_directory(Dir),
Chris@0: 	    file_base_name(Dir, Base),
Chris@0: 	    \+ hidden_base(Base),
Chris@0: 	    attach_dir(Dir, [Path|Visited]),
Chris@0: 	    fail ; true
Chris@0: 	).
Chris@0: 
Chris@0: hidden_base('CVS').
Chris@0: hidden_base('cvs').			% Windows
Chris@0: 
Chris@0: %%	process_manifest(+Location) is det.
Chris@0: %
Chris@0: %	Process a manifest file, registering  encountered namespaces and
Chris@0: %	creating clauses for library/3. No op if manifest was loaded and
Chris@0: %	not changed. Removes old data if the manifest was changed.
Chris@0: %	
Chris@0: %	@param	Location is either a path name or a URL.
Chris@0: 
Chris@0: process_manifest(Source) :-
Chris@0: 	(   file_name_to_url(Manifest0, Source)
Chris@0: 	->  absolute_file_name(Manifest0, Manifest)
Chris@0: 	;   Manifest = Source
Chris@0: 	),
Chris@0: 	source_time(Manifest, MT),
Chris@0: 	(   manifest(Manifest, Time),
Chris@0: 	    (	MT =< Time
Chris@0: 	    ->  !
Chris@0: 	    ;	retractall(manifest(Manifest, Time)),
Chris@0: 	        library_db(Id, URL, Facets),
Chris@0: 		memberchk(manifest(Manifest), Facets),
Chris@0: 		retractall(library_db(Id, URL, Facets)),
Chris@0: 		fail
Chris@0: 	    )
Chris@0: 	;   read_triples(Manifest, Triples),
Chris@0: 	    process_triples(Manifest, Triples),
Chris@0: 	    print_message(informational, rdf(manifest(loaded, Manifest))),
Chris@0: 	    assert(manifest(Manifest, MT))
Chris@0: 	).
Chris@0: 
Chris@0: process_triples(Manifest, Triples) :-
Chris@0: 	findall(ns(Mnemonic, NameSpace),
Chris@0: 		extract_namespace(Triples, Mnemonic, NameSpace),
Chris@0: 		NameSpaces),
Chris@0: 	findall(Ontology,
Chris@0: 		extract_ontology(Triples, Ontology), 
Chris@0: 		Ontologies),
Chris@0: 	maplist(define_namespace, NameSpaces),
Chris@0: 	maplist(assert_ontology(Manifest), Ontologies).
Chris@0: 
Chris@0: %%	extract_namespace(+Triples, -Mnemonic, -NameSpace)
Chris@0: %
Chris@0: %	True if Mnemonic is an abbreviation of NameSpace.
Chris@0: 
Chris@0: extract_namespace(Triples, Mnemonic, Namespace) :-
Chris@0: 	edge(Triples, Decl, lib:mnemonic, literal(Mnemonic)),
Chris@0: 	edge(Triples, Decl, lib:namespace, Namespace).
Chris@0: 
Chris@0: %%	extract_ontology(+Triples, -Ontology) is nondet.
Chris@0: %
Chris@0: %	Extract definition of an ontology
Chris@0: 
Chris@0: extract_ontology(Triples, library(Name, URL, Options)) :-
Chris@0: 	edge(Triples, URL, rdf:type, Type),
Chris@0: 	(   ontology_type(Type)
Chris@0: 	->  file_base_name(URL, BaseName),
Chris@0: 	    file_name_extension(Name, _, BaseName),
Chris@0: 	    findall(Facet, facet(Triples, URL, Facet), Options)
Chris@0: 	).
Chris@0: 
Chris@0: ontology_type(X) :-
Chris@0: 	(   rdf_equal(X, lib:'Ontology')
Chris@0: 	;   rdf_equal(X, lib:'Schema')
Chris@0: 	;   rdf_equal(X, lib:'Instances')
Chris@0: 	).
Chris@0: 
Chris@0: %%	facet(+Triples, +File, -Facet) is nondet.
Chris@0: %
Chris@0: %	Enumerate facets about File from   Triples. Facets are described
Chris@0: %	with rdf_library_index/2.
Chris@0: 
Chris@0: facet(Triples, File, title(Title)) :-
Chris@0: 	edge(Triples, File, dc:title, literal(Title)).
Chris@0: facet(Triples, File, version(Version)) :-
Chris@0: 	edge(Triples, File, owl:versionInfo, literal(Version)).
Chris@0: facet(Triples, File, comment(Comment)) :-
Chris@0: 	edge(Triples, File, rdfs:comment, literal(Comment)).
Chris@0: facet(Triples, File, base_uri(BaseURI)) :-
Chris@0: 	edge(Triples, File, lib:baseURI, BaseURI).
Chris@0: facet(Triples, File, claimed_source(Source)) :-
Chris@0: 	edge(Triples, File, lib:source, Source).
Chris@0: facet(Triples, File, blank_nodes(Mode)) :-
Chris@0: 	edge(Triples, File, lib:blankNodes, literal(Mode)),
Chris@0: 	must_be(oneof([share,noshare]), Mode).
Chris@0: facet(Triples, File, imports(ontology, Path)) :-
Chris@0: 	edge(Triples, File, owl:imports, Path).
Chris@0: facet(Triples, File, imports(schema, Path)) :-
Chris@0: 	edge(Triples, File, lib:schema, Path).
Chris@0: facet(Triples, File, imports(instances, Path)) :-
Chris@0: 	edge(Triples, File, lib:instances, Path).
Chris@0: facet(Triples, File, provides_ns(NS)) :-
Chris@0: 	edge(Triples, File, lib:providesNamespace, NSDecl),
Chris@0: 	edge(Triples, NSDecl, lib:namespace, NS).
Chris@0: facet(Triples, File, uses_ns(NS)) :-
Chris@0: 	edge(Triples, File, lib:usesNamespace, NSDecl),
Chris@0: 	edge(Triples, NSDecl, lib:namespace, NS).
Chris@0: facet(Triples, File, virtual) :-
Chris@0: 	edge(Triples, File, rdf:type, lib:'Virtual').
Chris@0: 
Chris@0: %%	edge(+Triples, ?S, ?P, ?O) is nondet.
Chris@0: %
Chris@0: %	Like rdf/3 over a list of Triples.
Chris@0: 
Chris@0: edge(Triples, S, P, O) :-
Chris@0: 	member(rdf(S,P,O), Triples).
Chris@0: 
Chris@0: %%	source_time(+Source, -Modified) is semidet.
Chris@0: %
Chris@0: %	Modified is the last modification time of Source.
Chris@0: %
Chris@0: %	@error	existence_error(Type, Source).
Chris@0: 
Chris@0: source_time(URL, Modified) :-
Chris@0: 	sub_atom(URL, 0, _, _, 'http://'), !,
Chris@0: 	http_open(URL, Stream,
Chris@0: 		  [ header(last_modified, Date),
Chris@0: 		    method(head)
Chris@0: 		  ]),
Chris@0: 	close(Stream),
Chris@0: 	Date \== '',
Chris@0: 	parse_time(Date, Modified).
Chris@0: source_time(URL, Modified) :-
Chris@0: 	file_name_to_url(File, URL), !,
Chris@0: 	time_file(File, Modified).
Chris@0: source_time(File, Modified) :-
Chris@0: 	time_file(File, Modified).
Chris@0: 
Chris@0: 
Chris@0: %%	read_triples(+File, -Triples) is det.
Chris@0: %
Chris@0: %	Read RDF/XML or Turtle file into a list of triples.
Chris@0: 
Chris@0: read_triples(File, Triples) :-
Chris@0: 	file_name_extension(_, rdf, File), !,
Chris@0: 	load_rdf(File, Triples).
Chris@0: read_triples(File, Triples) :-
Chris@0: 	file_name_extension(_, ttl, File), !,
Chris@0: 	rdf_load_turtle(File, Triples, []).
Chris@0: 
Chris@0: %%	is_manifest_file(+Path)
Chris@0: %
Chris@0: %	True if Path is the name of a manifest file.
Chris@0: 
Chris@0: is_manifest_file(Path) :-
Chris@0: 	file_base_name(Path, File),
Chris@0: 	downcase_atom(File, Lwr),
Chris@0: 	file_name_extension(Base, Ext, Lwr),
Chris@0: 	manifest_file(Base),
Chris@0: 	rdf_extension(Ext), !.
Chris@0: 	
Chris@0: manifest_file('Manifest').
Chris@0: manifest_file('manifest').
Chris@0: 
Chris@0: rdf_extension(ttl).
Chris@0: rdf_extension(rdf).
Chris@0: 
Chris@0: 
Chris@0: %%	assert_ontology(+Manifest, +Term:library(Name, File, Facets)) is det.
Chris@0: %
Chris@0: %	Add ontology to our library.
Chris@0: %	
Chris@0: %	@tbd	Proper behaviour of re-definition?
Chris@0: 
Chris@0: assert_ontology(Manifest, Term) :-
Chris@0: 	Term = library(Name, URL, Facets),
Chris@0: 	(   library(Name, _URL2, Facets2)
Chris@0: 	->  memberchk(manifest(Manifest2), Facets2),
Chris@0: 	    print_message(warning, rdf(redefined(Manifest, Name, Manifest2)))
Chris@0: 	;   true
Chris@0: 	),
Chris@0: 	assert(library_db(Name, URL,
Chris@0: 		       [ manifest(Manifest)
Chris@0: 		       | Facets
Chris@0: 		       ])).
Chris@0: 
Chris@0: 
Chris@0: %%	library(?Id, ?URL, ?Facets)
Chris@0: %
Chris@0: %	Access DB for library information.
Chris@0: 
Chris@0: library(Id, URL, Facets) :-
Chris@0: 	nonvar(URL),
Chris@0: 	canonical_url(URL, CanonicalURL),
Chris@0: 	library_db(Id, CanonicalURL, Facets).
Chris@0: library(Id, URL, Facets) :-
Chris@0: 	library_db(Id, URL, Facets).
Chris@0: 
Chris@0: %%	canonical_url(+URL, -CanonicalURL) is det.
Chris@0: %
Chris@0: %	Translate a URL into a  canonical   form.  Currently  deals with
Chris@0: %	file:// urls to take care of  filesystem properies such as being
Chris@0: %	case insensitive and symbolic names.
Chris@0: %	
Chris@0: %	@tbd	Generic URL handling should also strip ../, etc.
Chris@0: 
Chris@0: canonical_url(FileURL, URL) :-
Chris@0: 	file_name_to_url(File, FileURL), !,
Chris@0: 	absolute_file_name(File, Abs),
Chris@0: 	file_name_to_url(Abs, URL).
Chris@0: canonical_url(URL, URL).
Chris@0: 
Chris@0: %%	define_namespace(NS:ns(Mnemonic, Namespace)) is det.
Chris@0: %
Chris@0: %	Add namespace declaration for Mnemonic.
Chris@0: 
Chris@0: define_namespace(ns(Mnemonic, Namespace)) :-
Chris@0: 	debug(rdf_library, 'Adding NS ~w = ~q', [Mnemonic, Namespace]),
Chris@0: 	rdf_register_ns(Mnemonic, Namespace,
Chris@0: 			[
Chris@0: 			]).
Chris@0: 
Chris@0: 
Chris@0: 		 /*******************************
Chris@0: 		 *	      MESSAGES		*
Chris@0: 		 *******************************/
Chris@0: 
Chris@0: :- multifile
Chris@0: 	prolog:message/3.
Chris@0: 
Chris@0: prolog:message(rdf(no_manifest(Path))) -->
Chris@0: 	[ 'Directory ~w has no Manifest.{ttl,rdf} file'-[Path] ].
Chris@0: prolog:message(rdf(redefined(Manifest, Name, Manifest2))) -->
Chris@0: 	[ '~w: Ontology ~w already defined in ~w'-
Chris@0: 	  [Manifest, Name, Manifest2]
Chris@0: 	].
Chris@0: prolog:message(rdf(manifest(loaded, Manifest))) -->
Chris@0: 	[ 'Loaded RDF manifest ~w'-[Manifest]
Chris@0: 	].
Chris@0: prolog:message(rdf(load_conflict(C1, C2))) -->
Chris@0: 	[ 'Conflicting loads: ~p <-> ~p'-[C1, C2] ].
Chris@0: prolog:message(rdf(loading(Files, Threads))) -->
Chris@0: 	[ 'Loading ~D files using ~D threads ...'-[Files, Threads] ].
Chris@0: