1/*  Part of SWI-Prolog
    2
    3    Author:        Jan Wielemaker
    4    E-mail:        J.Wielemaker@vu.nl
    5    WWW:           http://www.swi-prolog.org
    6    Copyright (c)  2010-2014, University of Amsterdam
    7                              VU University Amsterdam
    8    All rights reserved.
    9
   10    Redistribution and use in source and binary forms, with or without
   11    modification, are permitted provided that the following conditions
   12    are met:
   13
   14    1. Redistributions of source code must retain the above copyright
   15       notice, this list of conditions and the following disclaimer.
   16
   17    2. Redistributions in binary form must reproduce the above copyright
   18       notice, this list of conditions and the following disclaimer in
   19       the documentation and/or other materials provided with the
   20       distribution.
   21
   22    THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
   23    "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
   24    LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
   25    FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
   26    COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
   27    INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
   28    BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
   29    LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
   30    CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
   31    LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
   32    ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
   33    POSSIBILITY OF SUCH DAMAGE.
   34*/
   35
   36:- module(rdf_library,
   37          [ rdf_attach_library/1,       % +Dir
   38            rdf_load_library/1,         % +Ontology
   39            rdf_load_library/2,         % +Ontology, +Options
   40            rdf_list_library/0,
   41            rdf_list_library/1,         % +Ontology
   42            rdf_list_library/2,         % +Ontology, +Options
   43            rdf_library_source/2,       % +Ontology, -SourceURL
   44            rdf_library_index/2,        % ?Id, ?Facet
   45            rdf_current_manifest/1      % -Manifest
   46          ]).   47:- use_module(library(semweb/rdf_db)).   48:- use_module(library(semweb/turtle)).   49:- use_module(library(rdf)).   50:- use_module(library(lists)).   51:- use_module(library(option)).   52:- use_module(library(debug)).   53:- use_module(library(error)).   54:- use_module(library(pairs)).   55:- use_module(library(date)).   56:- use_module(library(uri)).   57:- use_module(library(http/http_open)).   58:- use_module(library(thread)).   59:- use_module(library(apply)).   60:- use_module(library(solution_sequences)).   61
   62:- predicate_options(rdf_list_library/2, 2,
   63                     [ indent(atom),
   64                       show_graph(boolean),
   65                       show_source(boolean),
   66                       show_virtual(boolean)
   67                     ]).   68:- predicate_options(rdf_load_library/2, 2,
   69                     [ concurrent(positive_integer),
   70                       import(boolean),
   71                       load(boolean),
   72                       base_uri(atom),
   73                       claimed_source(atom),
   74                       not_found(oneof([error,warning,silent]))
   75                     ]).   76
   77/** <module> RDF Library Manager
   78
   79This module manages an ontology library. Such   a  library consists of a
   80directory with manifest files named =|Manifest.rdf|= or =|Manifest.ttl|=
   81(Turtle). The manifest files define ontologies  appearing in the library
   82as well as namespace mnemonics and dependencies.
   83
   84The typical usage scenario is
   85
   86==
   87?- rdf_attach_library('/some/directory').
   88?- rdf_load_library(my_ontology).
   89==
   90
   91@tbd    Add caching info
   92@tbd    Allow using Manifests on HTTP servers
   93@author Jan Wielemaker
   94*/
   95
   96:- rdf_register_ns(lib,  'http://www.swi-prolog.org/rdf/library/').   97:- rdf_register_ns(void, 'http://rdfs.org/ns/void#').   98:- rdf_register_ns(vann, 'http://purl.org/vocab/vann/').   99
  100:- dynamic
  101    manifest/2,                     % Path, Time
  102    library_db/3.                   % Name, URL, Facets
  103
  104%       Force compile-time namespace expansion
  105
  106:- rdf_meta
  107    edge(+, r,r,o).  108
  109                 /*******************************
  110                 *            LOADING           *
  111                 *******************************/
  112
  113%!  rdf_load_library(+Id) is det.
  114%!  rdf_load_library(+Id, +Options) is det.
  115%
  116%   Load ontologies from the  library.  A   library  must  first  be
  117%   attached using rdf_attach_library/1.  Defined Options are:
  118%
  119%           * import(Bool)
  120%           If =true= (default), also load ontologies that are
  121%           explicitely imported.
  122%
  123%           * base_uri(URI)
  124%           BaseURI used for loading RDF.  Local definitions in
  125%           ontologies overrule this option.
  126%
  127%           * claimed_source(URL)
  128%           URL from which we claim to have loaded the data.
  129%
  130%           * not_found(+Level)
  131%           The system does a pre-check for the existence of
  132%           all references RDF databases.  If Level is =error=
  133%           it reports missing databases as an error and fails.
  134%           If =warning= it prints them, but continues.  If
  135%           =silent=, no checks are preformed.  Default is =error=.
  136%
  137%           * concurrent(Threads)
  138%           Perform the load concurrently using N threads.  If not
  139%           specified, the number is determined by
  140%           guess_concurrency/2.
  141%
  142%           * load(+Bool)
  143%           If =false=, to all the preparation, but do not execute
  144%           the actual loading.  See also rdf_list_library/2.
  145
  146rdf_load_library(Id) :-
  147    rdf_load_library(Id, []).
  148
  149rdf_load_library(Id, Options) :-
  150    cleaned_load_commands(Id, Cmds, Options),
  151    (   option(concurrent(Threads), Options)
  152    ->  true
  153    ;   guess_concurrency(Cmds, Threads)
  154    ),
  155    length(Cmds, NSources),
  156    print_message(informational, rdf(loading(NSources, Threads))),
  157    (   option(load(true), Options, true)
  158    ->  concurrent(Threads, Cmds, [])
  159    ;   true
  160    ).
  161
  162%!  rdf_library_source(+Id, -Source) is nondet.
  163%
  164%   True of Source is the URL that is  part of the given library Id.
  165%   This predicate finds all indirect   dependencies.  It does _not_
  166%   check whether the source exists or is valid.
  167%
  168%   @see uri_file_name/2 for converting file:// URLs to a filename.
  169
  170rdf_library_source(Id, Source) :-
  171    cleaned_load_commands(Id, Cmds,
  172                          [ import(true),
  173                            not_found(silent)
  174                          ]),
  175    member(rdf_load(Source, _), Cmds).
  176
  177
  178cleaned_load_commands(Id, Cmds, Options) :-
  179    load_commands(Id, Options, Pairs),
  180    pairs_values(Pairs, Commands),
  181    list_to_set(Commands, Cmds2),
  182    delete_virtual(Cmds2, Cmds3),
  183    find_conflicts(Cmds3),
  184    check_existence(Cmds3, Cmds, Options).
  185
  186delete_virtual([], []).
  187delete_virtual([virtual(_)|T0], T) :-
  188    !,
  189    delete_virtual(T0, T).
  190delete_virtual([H|T0], [H|T]) :-
  191    delete_virtual(T0, T).
  192
  193
  194%!  find_conflicts(+LoadCommands) is semidet.
  195%
  196%   Find possibly conflicting options for loading the same source
  197
  198find_conflicts(Commands) :-
  199    no_source_with_different_options(Commands),
  200    no_sources_in_same_graph(Commands).
  201
  202%!  no_source_with_different_options(+Commands) is semidet.
  203%
  204%   True if there are not multiple calls to load the same graph, but
  205%   with  different  load-options.  Prints  a    warning  and  fails
  206%   otherwise.
  207
  208no_source_with_different_options(Commands) :-
  209    sort(Commands, Cmds),
  210    conflicts(Cmds, Conflicts),
  211    report_conflicts(Conflicts),
  212    Conflicts == [].
  213
  214conflicts([], []).
  215conflicts([C1, C2|T0], [C1-C2|T]) :-
  216    conflict(C1, C2),
  217    !,
  218    conflicts([C2|T0], T).
  219conflicts([_|T0], T) :-
  220    conflicts(T0, T).
  221
  222conflict(rdf_load(Src, Options1), rdf_load(Src, Options2)) :-
  223    sort(Options1, S1),
  224    sort(Options2, S2),
  225    S1 \== S2.
  226
  227report_conflicts([]).
  228report_conflicts([C1-C2|T]) :-
  229    print_message(warning, rdf(load_conflict(C1,C2))),
  230    report_conflicts(T).
  231
  232%!  no_sources_in_same_graph(+Commands) is semidet.
  233%
  234%   True if there are not two load   commands  referring to the same
  235%   graph.
  236
  237no_sources_in_same_graph(Commands) :-
  238    map_list_to_pairs(command_graph, Commands, Keyed),
  239    keysort(Keyed, KeySorted),
  240    group_pairs_by_key(KeySorted, SourcesByGraph),
  241    (   member(Graph-Sources, SourcesByGraph),
  242        Sources = [_,_|_]
  243    ->  forall(( member(Graph-Sources, SourcesByGraph),
  244                 Sources = [_,_|_]
  245               ),
  246               print_message(error,
  247                             rdf(multiple_source_for_graph(Graph, Sources)))),
  248        fail
  249    ;   true
  250    ).
  251
  252command_graph(rdf_load(_, Options), Graph) :-
  253    option(graph(Graph), Options),
  254    !.
  255command_graph(rdf_load(URL, _), URL) :- !.
  256command_graph(_, _).                    % Other command.  Each variable it its own key
  257
  258
  259%!  check_existence(+CommandsIn, -Commands, +Options) is det.
  260%
  261%   Report existence errors. Fail if at   least  one source does not
  262%   exist. and the not_found level is not =silent=.
  263%
  264%   @error existence_error(urls, ListOfUrls)
  265
  266check_existence(CommandsIn, Commands, Options) :-
  267    option(not_found(Level), Options, error),
  268    must_be(oneof([error,warning,silent]), Level),
  269    (   Level == silent
  270    ->  Commands = CommandsIn
  271    ;   missing_urls(CommandsIn, Commands, Missing),
  272        (   Missing == []
  273        ->  true
  274        ;   Level == warning
  275        ->  report_missing(Missing, Level)
  276        ;   existence_error(urls, Missing)
  277        )
  278    ).
  279
  280
  281missing_urls([], [], []).
  282missing_urls([H|T0], Cmds, Missing) :-
  283    H = rdf_load(URL, _),
  284    (   catch(exists_url(URL, _Ext), error(existence_error(_,_), _), fail)
  285    ->  Cmds = [H|T],
  286        missing_urls(T0, T, Missing)
  287    ;   Missing = [URL|T],
  288        missing_urls(T0, Cmds, T)
  289    ).
  290
  291report_missing([], _).
  292report_missing([H|T], Level) :-
  293    print_message(Level, error(existence_error(url, H), _)),
  294    report_missing(T, Level).
  295
  296%!  guess_concurrency(+Commands, -Threads) is det.
  297%
  298%   How much concurrency to use? Set to   the  number of CPUs if all
  299%   input comes from  files  or  5   if  network  based  loading  is
  300%   demanded.
  301
  302guess_concurrency(Commands, Threads) :-
  303    count_uris(Commands, FileURLs, OtherURLs),
  304    (   FileURLs > 0
  305    ->  (   current_prolog_flag(cpu_count, CPUs)
  306        ->  true
  307        ;   CPUs = 1
  308        ),
  309        FileThreads is min(FileURLs, CPUs)
  310    ;   FileThreads = 0
  311    ),
  312    (   OtherURLs > 0
  313    ->  OtherThreads is min(5, OtherURLs)
  314    ;   OtherThreads = 0
  315    ),
  316    Threads is FileThreads + OtherThreads.
  317
  318count_uris([], 0, 0).
  319count_uris([rdf_load(URL, _)|T], F, NF) :-
  320    count_uris(T, F0, NF0),
  321    (   web_url(URL)
  322    ->  NF is NF0 + 1,
  323        F = F0
  324    ;   F is F0 + 1,
  325        NF = NF0
  326    ).
  327
  328
  329%!  load_commands(+Id, +Options, -Pairs:list(Level-Command)) is det.
  330%
  331%   Commands are the RDF commands to execute for rdf_load_library/2.
  332%   Splitting  in  command  collection  and   execution  allows  for
  333%   concurrent execution as well  as   forward  checking of possible
  334%   problems.
  335%
  336%   @tbd    Fix poor style; avoid assert/retract.
  337
  338:- thread_local
  339    command/2.  340
  341load_commands(Id, Options, Commands) :-
  342    retractall(command(_,_)),
  343    rdf_update_library_index,
  344    dry_load(Id, 1, Options),
  345    findall(Level-Cmd, retract(command(Level, Cmd)), Commands).
  346
  347dry_load(Id, Level, Options) :-
  348    (   library(Id, File, Facets)
  349    ->  merge_base_uri(Facets, Options, Options1),
  350        merge_source(Facets, Options1, Options2),
  351        merge_blanks(Facets, Options2, Options3),
  352        merge_format(Facets, Options3, Options4),
  353        (   \+ memberchk(virtual, Facets)
  354        ->  load_options(Options4, File, RdfOptions),
  355            assert(command(Level, rdf_load(File, RdfOptions)))
  356        ;   assert(command(Level, virtual(File)))
  357        ),
  358        (   option(import(true), Options, true)
  359        ->  Level1 is Level + 1,
  360            forall(member(imports(Type, Import), Facets),
  361                   import(Import, Level1, [type(Type)|Options4]))
  362        ;   true
  363        )
  364    ;   existence_error(ontology, Id)
  365    ).
  366
  367merge_base_uri(Facets, Options0, Options) :-
  368    (   option(base_uri(Base), Facets)
  369    ->  exclude(name_option(base_uri), Options0, Options1),
  370        Options = [base_uri(Base)|Options1]
  371    ;   Options = Options0
  372    ).
  373
  374merge_source(Facets, Options0, Options) :-
  375    (   option(claimed_source(Base), Facets)
  376    ->  exclude(name_option(claimed_source), Options0, Options1),
  377        Options = [claimed_source(Base)|Options1]
  378    ;   Options = Options0
  379    ).
  380
  381merge_blanks(Facets, Options0, Options) :-
  382    (   option(blank_nodes(Share), Facets)
  383    ->  exclude(name_option(blank_nodes), Options0, Options1),
  384        Options = [blank_nodes(Share)|Options1]
  385    ;   Options = Options0
  386    ).
  387
  388merge_format(Facets, Options0, Options) :-
  389    (   option(format(Format), Facets)
  390    ->  exclude(name_option(format), Options0, Options1),
  391        Options = [format(Format)|Options1]
  392    ;   Options = Options0
  393    ).
  394
  395name_option(Name, Term) :-
  396    functor(Term, Name, 1).
  397
  398load_options(Options, File, RDFOptions) :-
  399    findall(O, load_option(Options, File, O), RDFOptions).
  400
  401load_option(Options, File, graph(Source)) :-
  402    option(claimed_source(Source0), Options),
  403    (   sub_atom(Source0, _, _, 0, /)
  404    ->  file_base_name(File, Base),
  405        atom_concat(Source0, Base, Source)
  406    ;   atom_concat(Source, #, Source0)
  407    ->  true
  408    ).
  409load_option(Options, File, base_uri(BaseURI)) :-
  410    option(base_uri(Base0), Options),
  411    sub_atom(/, _, _, 0, Base0),
  412    atom_concat(Base0, File, BaseURI).
  413load_option(Options, _File, blank_nodes(Share)) :-
  414    option(blank_nodes(Share), Options).
  415load_option(Options, _File, format(Format)) :-
  416    option(format(Format), Options).
  417
  418%!  import(+URL, +Level, +Options) is det.
  419
  420import(Path, Level, Options) :-
  421    option(type(data_dump), Options),
  422    !,
  423    load_options(Options, Path, RdfOptions),
  424    assert(command(Level, rdf_load(Path, RdfOptions))).
  425import(Path, Level, Options) :-
  426    (   (   library(Id, Path, _)
  427        ->  true
  428        ;   manifest_for_path(Path, Manifest),
  429            catch(exists_url(Manifest, _Ext), _, fail)
  430        ->  process_manifest(Manifest),
  431            library(Id, Path, _)
  432        )
  433    ->  dry_load(Id, Level, Options)
  434    ;   load_options(Options, Path, RdfOptions),
  435        assert(command(Level, rdf_load(Path, RdfOptions)))
  436    ).
  437
  438manifest_for_path(URL, Manifest) :-
  439    file_directory_name(URL, Parent),
  440    manifest_file(Base),
  441    rdf_extension(Ext),
  442    atomic_list_concat([Parent, /, Base, '.', Ext], Manifest).
  443
  444%!  rdf_list_library(+Id) is det.
  445%!  rdf_list_library(+Id, +Options) is det.
  446%
  447%   Print library dependency tree to the terminal.  Options include
  448%   options for rdf_load_library/2 and
  449%
  450%           * show_source(+Boolean)
  451%           If =true= (default), show location we are loading
  452%
  453%           * show_graph(+Boolean)
  454%           If =true= (default =false=), show name of graph
  455%
  456%           * show_virtual(+Boolean)
  457%           If =false= (default =true=), do not show virtual
  458%           repositories.
  459%
  460%           * indent(Atom)
  461%           Atom repeated for indentation levels
  462
  463rdf_list_library(Id) :-
  464    rdf_list_library(Id, []).
  465rdf_list_library(Id, Options) :-
  466    load_commands(Id, Options, Commands),
  467    maplist(print_load(Options), Commands).
  468
  469print_load(Options, _Level-virtual(_)) :-
  470    option(show_virtual(false), Options),
  471    !.
  472print_load(Options, Level-Command) :-
  473    option(indent(Indent), Options, '. '),
  474    forall(between(2, Level, _), format(Indent)),
  475    print_command(Command, Options),
  476    format('~N').
  477
  478print_command(virtual(URL), _Options) :-
  479    format('<~w>', [URL]).
  480print_command(rdf_load(URL), Options) :-
  481    print_command(rdf_load(URL, []), Options).
  482print_command(rdf_load(URL, RDFOptions), Options) :-
  483    (   option(show_source(true), Options, true)
  484    ->  format('~w', [URL]),
  485        (   option(blank_nodes(noshare), RDFOptions)
  486        ->  format(' <not shared>')
  487        ;   true
  488        ),
  489        (   exists_url(URL, Ext)
  490        ->  (   Ext == ''
  491            ->  true
  492            ;   format('[.~w]', [Ext])
  493            )
  494        ;   format(' [NOT FOUND]')
  495        )
  496    ;   true
  497    ),
  498    (   option(show_graph(true), Options, false),
  499        option(graph(Base), RDFOptions)
  500    ->  format('~N\tSource: ~w', [Base])
  501    ;   true
  502    ).
  503
  504exists_url(URL, Ext) :-
  505    uri_file_name(URL, Path),
  506    !,
  507    add_storage_extension(Path, Ext, PathEx),
  508    access_file(PathEx, read),
  509    !.
  510exists_url(URL, Ext) :-
  511    uri_components(URL, Components),
  512    uri_data(scheme, Components, Scheme),
  513    atom(Scheme),
  514    url_scheme(Scheme),
  515    add_storage_extension(URL, Ext, URLEx),
  516    catch(http_open(URLEx, Stream, [ method(head) ]), _, fail),
  517    !,
  518    close(Stream).
  519
  520:- multifile
  521    rdf_db:rdf_storage_encoding/2.  522
  523add_storage_extension(File, '', File).
  524add_storage_extension(File, Ext, FileEx) :-
  525    rdf_db:rdf_storage_encoding(Ext, _Format),
  526    \+ file_name_extension(_, Ext, File),
  527    file_name_extension(File, Ext, FileEx).
  528
  529url_scheme(http).
  530url_scheme(https).
  531
  532
  533%!  rdf_list_library
  534%
  535%   Prints known RDF library identifiers to current output.
  536
  537rdf_list_library :-
  538    rdf_update_library_index,
  539    (   rdf_library_index(Id, title(TitleLiteral)),
  540        plain_string(TitleLiteral, Title),
  541        format('~w ~t~20|~w', [Id, Title]),
  542        (   rdf_library_index(Id, version(Version))
  543        ->  format(' (version ~w)', [Version])
  544        ;   true
  545        ),
  546        nl,
  547        fail
  548    ;   true
  549    ).
  550
  551plain_string(String, String) :-
  552    atomic(String),
  553    !.
  554plain_string(lang(en, String), String) :- !.
  555plain_string(lang(_, String), String) :- !.
  556plain_string(type(_, String), String) :- !.
  557
  558%!  rdf_library_index(?Id, ?Facet) is nondet.
  559%
  560%   Query the content of the library.  Defined facets are:
  561%
  562%           * source(URL)
  563%           Location from which to load the ontology
  564%
  565%           * title(Atom)
  566%           Title used for the ontology
  567%
  568%           * comment(Atom)
  569%           Additional comments for the ontology
  570%
  571%           * version(Atom)
  572%           Version information on the ontology
  573%
  574%           * imports(Type, URL)
  575%           URLs needed by this ontology. May succeed multiple
  576%           times.  Type is one of =ontology=, =schema= or =instances=.
  577%
  578%           * base_uri(BaseURI)
  579%           Base URI to use when loading documents. If BaseURI
  580%           ends in =|/|=, the actual filename is attached.
  581%
  582%           * claimed_source(Source)
  583%           URL from which we claim to have loaded the RDF. If
  584%           Source ends in =|/|=, the actual filename is
  585%           attached.
  586%
  587%           * blank_nodes(Share)
  588%           Defines how equivalent blank nodes are handled, where
  589%           Share is one of =share= or =noshare=.  Default is to
  590%           share.
  591%
  592%           * format(Format)
  593%           Format of the resource.  Can be used to overrule
  594%           if the format as derived from the HTTP content type
  595%           is wrong.
  596%
  597%           * provides_ns(URL)
  598%           Ontology provides definitions in the namespace URL.
  599%           The formal definition of this is troublesome, but in
  600%           practice it means the ontology has triples whose
  601%           subjects are in the given namespace.
  602%
  603%           * uses_ns(URL)
  604%           The ontology depends on the given namespace.  Normally
  605%           means it contains triples that have predicates or
  606%           objects in the given namespace.
  607%
  608%           * manifest(URL)
  609%           URL of the manifest in which this ontology is defined.
  610%
  611%           * virtual
  612%           Entry is virtual (cannot be loaded)
  613
  614rdf_library_index(Id, Facet) :-
  615    library(Id, Path, Facets),
  616    (   Facet = source(Path)
  617    ;   member(Facet, Facets)
  618    ).
  619
  620
  621                 /*******************************
  622                 *      MANIFEST PROCESSING     *
  623                 *******************************/
  624
  625%!  rdf_attach_library(+Source)
  626%
  627%   Attach manifest from Source.  Source is one of
  628%
  629%           * URL
  630%           Load single manifest from this URL
  631%           * File
  632%           Load single manifest from this file
  633%           * Directory
  634%           Scan all subdirectories and load all =|Manifest.ttl|= or
  635%           =|Manifest.rdf|= found.  If Directory is a path-alias
  636%           (e.g., ontology(.)), _all_ referenced directories are
  637%           scanned for manifest files.
  638%
  639%   Encountered namespaces are registered   using rdf_register_ns/2.
  640%   Encountered ontologies are added to the index. If a manifest was
  641%   already loaded it will be reloaded  if the modification time has
  642%   changed.
  643
  644rdf_attach_library(URL) :-
  645    atom(URL),
  646    uri_is_global(URL),
  647    \+ is_absolute_file_name(URL),   % avoid interpreting C: as a schema
  648    !,
  649    process_manifest(URL).
  650rdf_attach_library(File) :-
  651    absolute_file_name(File, Path,
  652                       [ extensions([rdf,ttl]),
  653                         access(read),
  654                         file_errors(fail)
  655                       ]),
  656    !,
  657    process_manifest(Path).
  658rdf_attach_library(Dir) :-
  659    forall(absolute_file_name(Dir, Path,
  660                              [ file_type(directory),
  661                                access(read),
  662                                solutions(all)
  663                              ]),
  664           attach_dir(Path, [])).
  665
  666
  667%!  rdf_update_library_index
  668%
  669%   Reload all Manifest files.
  670
  671rdf_update_library_index :-
  672    forall(manifest(Location, _Time),
  673           process_manifest(Location)).
  674
  675attach_dir(Path, Visited) :-
  676    memberchk(Path, Visited),
  677    !.
  678attach_dir(Path, Visited) :-
  679    atom_concat(Path, '/*', Pattern),
  680    expand_file_name(Pattern, Members),
  681    (   manifest_file(MBase),
  682        rdf_extension(Ext),
  683        atomic_list_concat([Path, /, MBase, '.', Ext], Manifest),
  684        exists_file(Manifest)
  685    ->  process_manifest(Manifest)
  686    ;   print_message(silent, rdf(no_manifest(Path)))
  687    ),
  688    (   member(Dir, Members),
  689        exists_directory(Dir),
  690        file_base_name(Dir, Base),
  691        \+ hidden_base(Base),
  692        attach_dir(Dir, [Path|Visited]),
  693        fail ; true
  694    ).
  695
  696hidden_base('CVS').
  697hidden_base('cvs').                     % Windows
  698
  699%!  process_manifest(+Location) is det.
  700%
  701%   Process a manifest file, registering  encountered namespaces and
  702%   creating clauses for library/3. No op if manifest was loaded and
  703%   not changed. Removes old data if the manifest was changed.
  704%
  705%   @param  Location is either a path name or a URL.
  706
  707process_manifest(Source) :-
  708    (   web_url(Source)
  709    ->  uri_normalized(Source, Manifest)
  710    ;   uri_file_name(Source, Manifest0)
  711    ->  absolute_file_name(Manifest0, ManifestFile),
  712        uri_file_name(Manifest, ManifestFile)
  713    ;   absolute_file_name(Source, ManifestFile),
  714        uri_file_name(Manifest, ManifestFile)
  715    ),                              % Manifest is a canonical URI
  716    source_time(Manifest, MT),
  717    (   manifest(Manifest, Time),
  718        (   MT =< Time
  719        ->  !
  720        ;   retractall(manifest(Manifest, Time)),
  721            library_db(Id, URL, Facets),
  722            memberchk(manifest(Manifest), Facets),
  723            retractall(library_db(Id, URL, Facets)),
  724            fail
  725        )
  726    ;   read_triples(Manifest, Triples),
  727        process_triples(Manifest, Triples),
  728        print_message(informational, rdf(manifest(loaded, Manifest))),
  729        assert(manifest(Manifest, MT))
  730    ).
  731
  732process_triples(Manifest, Triples) :-
  733    findall(ns(Mnemonic, NameSpace),
  734            extract_namespace(Triples, Mnemonic, NameSpace),
  735            NameSpaces),
  736    findall(Ontology,
  737            extract_ontology(Triples, Ontology),
  738            Ontologies),
  739    maplist(define_namespace, NameSpaces),
  740    maplist(assert_ontology(Manifest), Ontologies).
  741
  742%!  extract_namespace(+Triples, -Mnemonic, -NameSpace)
  743%
  744%   True if Mnemonic is an abbreviation of NameSpace.
  745
  746extract_namespace(Triples, Mnemonic, Namespace) :-
  747    edge(Triples, Decl, lib:mnemonic, literal(Mnemonic)),
  748    edge(Triples, Decl, lib:namespace, Namespace).
  749extract_namespace(Triples, Mnemonic, Namespace) :-
  750    edge(Triples, Decl, vann:preferredNamespacePrefix, literal(Mnemonic)),
  751    edge(Triples, Decl, vann:preferredNamespaceUri, literal(Namespace)).
  752
  753%!  extract_ontology(+Triples, -Ontology) is nondet.
  754%
  755%   Extract definition of an ontology
  756
  757extract_ontology(Triples, library(Name, URL, Options)) :-
  758    distinct(URL, ontology(Triples, URL)),
  759    file_base_name(URL, BaseName),
  760    file_name_extension(Name, _, BaseName),
  761    findall(Facet, facet(Triples, URL, Facet), Options0),
  762    sort(Options0, Options1),
  763    keep_specialized_facets(Options1, Options).
  764
  765ontology(Triples, URL) :-
  766    edge(Triples, URL, rdf:type, Type),
  767    ontology_type(Type).
  768
  769keep_specialized_facets(All, Special) :-
  770    exclude(more_general(All), All, Special).
  771
  772more_general(All, Facet) :-
  773    generalized(Facet, Special),
  774    memberchk(Special, All).
  775
  776generalized(imports(ontology, Path), imports(Other, Path)) :-
  777    dif(Other, ontology).
  778
  779ontology_type(X) :-
  780    (   rdf_equal(X, lib:'Ontology')
  781    ;   rdf_equal(X, lib:'Schema')
  782    ;   rdf_equal(X, lib:'Instances')
  783    ;   rdf_equal(X, void:'Dataset')
  784    ;   rdf_equal(X, void:'Linkset')
  785    ).
  786
  787%!  facet(+Triples, +File, -Facet) is nondet.
  788%
  789%   Enumerate facets about File from   Triples. Facets are described
  790%   with rdf_library_index/2.
  791
  792facet(Triples, File, title(Title)) :-
  793    edge(Triples, File, dcterms:title, literal(Title)).
  794facet(Triples, File, version(Version)) :-
  795    edge(Triples, File, owl:versionInfo, literal(Version)).
  796facet(Triples, File, comment(Comment)) :-
  797    edge(Triples, File, rdfs:comment, literal(Comment)).
  798facet(Triples, File, base_uri(BaseURI)) :-
  799    edge(Triples, File, lib:baseURI, BaseURI).
  800facet(Triples, File, claimed_source(Source)) :-
  801    edge(Triples, File, lib:source, Source).
  802facet(Triples, File, format(Format)) :-
  803    edge(Triples, File, lib:format, literal(Format)).
  804facet(Triples, File, blank_nodes(Mode)) :-
  805    edge(Triples, File, lib:blankNodes, literal(Mode)),
  806    must_be(oneof([share,noshare]), Mode).
  807facet(Triples, File, imports(ontology, Path)) :-
  808    edge(Triples, File, owl:imports, Path).
  809facet(Triples, File, imports(schema, Path)) :-
  810    edge(Triples, File, lib:schema, Path).
  811facet(Triples, File, imports(instances, Path)) :-
  812    edge(Triples, File, lib:instances, Path).
  813facet(Triples, File, imports(subset, Path)) :-
  814    edge(Triples, File, void:subset, Path).
  815facet(Triples, File, imports(data_dump, Path)) :-
  816    edge(Triples, File, void:dataDump, Path).
  817facet(Triples, File, provides_ns(NS)) :-
  818    edge(Triples, File, lib:providesNamespace, NSDecl),
  819    edge(Triples, NSDecl, lib:namespace, NS).
  820facet(Triples, File, uses_ns(NS)) :-
  821    edge(Triples, File, lib:usesNamespace, NSDecl),
  822    edge(Triples, NSDecl, lib:namespace, NS).
  823facet(Triples, File, virtual) :-
  824    (   edge(Triples, File, rdf:type, lib:'Virtual')
  825    ;   edge(Triples, File, rdf:type, void:'Dataset')
  826    ;   edge(Triples, File, rdf:type, void:'Linkset')
  827    ) -> true.
  828
  829%!  edge(+Triples, ?S, ?P, ?O) is nondet.
  830%
  831%   Like rdf_has/3 over a list of Triples.
  832
  833edge(Triples, S, P, O) :-
  834    nonvar(P),
  835    !,
  836    sub_p(SubP, P),
  837    member(rdf(S,SubP,O), Triples).
  838edge(Triples, S, P, O) :-
  839    member(rdf(S,SubP,O), Triples),
  840    sub_p(SubP, P).
  841
  842sub_p(P, P).
  843sub_p(Sub, P) :-
  844    (   nonvar(Sub)
  845    ->  sub_property_of(Sub, Sub1),
  846        sub_p(Sub1, P)
  847    ;   sub_property_of(Sub1, P),
  848        sub_p(Sub, Sub1)
  849    ).
  850
  851:- rdf_meta
  852    sub_property_of(r,r).  853
  854sub_property_of(void:subset,         owl:imports).
  855sub_property_of(dcterms:description, rdfs:comment).
  856sub_property_of(void:dataDump,       owl:imports).
  857sub_property_of(dc:title,            dcterms:title).
  858
  859%!  source_time(+Source, -Modified) is semidet.
  860%
  861%   Modified is the last modification time of Source.
  862%
  863%   @error  existence_error(Type, Source).
  864
  865source_time(URL, Modified) :-
  866    web_url(URL),
  867    !,
  868    http_open(URL, Stream,
  869              [ header(last_modified, Date),
  870                method(head)
  871              ]),
  872    close(Stream),
  873    Date \== '',
  874    parse_time(Date, Modified).
  875source_time(URL, Modified) :-
  876    uri_file_name(URL, File),
  877    !,
  878    time_file(File, Modified).
  879source_time(File, Modified) :-
  880    time_file(File, Modified).
  881
  882web_url(URL) :-
  883    sub_atom(URL, 0, _, _, 'http://').
  884
  885
  886%!  read_triples(+URL, -Triples) is det.
  887%
  888%   Read RDF/XML or Turtle file into a list of triples.
  889
  890read_triples(FileURL, Triples) :-
  891    uri_file_name(FileURL, File),
  892    !,
  893    (   file_name_extension(_, rdf, File)
  894    ->  load_rdf(File, Triples)
  895    ;   rdf_load_turtle(File, Triples, [])
  896    ).
  897read_triples(HTTPURL, Triples) :-
  898    file_name_extension(_, Ext, HTTPURL),
  899    setup_call_cleanup(
  900        http_open(HTTPURL, In, []),
  901        stream_triples(In, Ext, Triples),
  902        close(In)).
  903
  904stream_triples(Stream, rdf, Triples) :-
  905    load_rdf(stream(Stream), Triples).
  906stream_triples(Stream, ttl, Triples) :-
  907    rdf_load_turtle(stream(Stream), Triples, []).
  908
  909
  910manifest_file('void').                  % make order optional?
  911manifest_file('Manifest').
  912manifest_file('manifest').
  913
  914rdf_extension(ttl).
  915rdf_extension(rdf).
  916
  917
  918%!  assert_ontology(+Manifest, +Term:library(Name, File, Facets)) is det.
  919%
  920%   Add ontology to our library.
  921%
  922%   @tbd    Proper behaviour of re-definition?
  923
  924assert_ontology(Manifest, Term) :-
  925    Term = library(Name, URL, Facets),
  926    (   library(Name, _URL2, Facets2)
  927    ->  memberchk(manifest(Manifest2), Facets2),
  928        print_message(warning, rdf(redefined(Manifest, Name, Manifest2)))
  929    ;   true
  930    ),
  931    assert(library_db(Name, URL,
  932                   [ manifest(Manifest)
  933                   | Facets
  934                   ])).
  935
  936
  937%!  library(?Id, ?URL, ?Facets)
  938%
  939%   Access DB for library information.
  940
  941library(Id, URL, Facets) :-
  942    nonvar(URL),
  943    normalize_url(URL, CanonicalURL),
  944    library_db(Id, CanonicalURL, Facets).
  945library(Id, URL, Facets) :-
  946    library_db(Id, URL, Facets).
  947
  948%!  normalize_url(+URL, -Normalized)
  949%
  950%   Like uri_normalized/2, but we  also   need  (platform dependent)
  951%   filename canonization.
  952
  953normalize_url(URL, CanonicalURL) :-
  954    uri_file_name(URL, File),
  955    !,
  956    absolute_file_name(File, CanFile),
  957    uri_file_name(CanonicalURL, CanFile).
  958normalize_url(URL, CanonicalURL) :-
  959    uri_normalized(URL, CanonicalURL).
  960
  961%!  define_namespace(NS:ns(Mnemonic, Namespace)) is det.
  962%
  963%   Add namespace declaration for Mnemonic.
  964
  965define_namespace(ns(Mnemonic, Namespace)) :-
  966    debug(rdf_library, 'Adding NS ~w = ~q', [Mnemonic, Namespace]),
  967    rdf_register_ns(Mnemonic, Namespace,
  968                    [
  969                        ]).
  970
  971%!  rdf_current_manifest(-URL) is nondet.
  972%
  973%   True if URL is the URL of a currently loaded manifest file.
  974
  975rdf_current_manifest(URL) :-
  976    manifest(URL, _Time).
  977
  978
  979
  980                 /*******************************
  981                 *            MESSAGES          *
  982                 *******************************/
  983
  984:- multifile
  985    prolog:message/3.  986
  987prolog:message(rdf(no_manifest(Path))) -->
  988    [ 'Directory ~w has no Manifest.{ttl,rdf} file'-[Path] ].
  989prolog:message(rdf(redefined(Manifest, Name, Manifest2))) -->
  990    [ '~w: Ontology ~w already defined in ~w'-
  991      [Manifest, Name, Manifest2]
  992    ].
  993prolog:message(rdf(manifest(loaded, Manifest))) -->
  994    [ 'Loaded RDF manifest ~w'-[Manifest]
  995    ].
  996prolog:message(rdf(load_conflict(C1, C2))) -->
  997    [ 'Conflicting loads: ~p <-> ~p'-[C1, C2] ].
  998prolog:message(rdf(multiple_source_for_graph(Graph, Sources))) -->
  999    [ 'Multiple sources for graph ~p:'-[Graph] ],
 1000    sources(Sources).
 1001prolog:message(rdf(loading(Files, Threads))) -->
 1002    [ 'Loading ~D files using ~D threads ...'-[Files, Threads] ].
 1003
 1004sources([]) --> [].
 1005sources([rdf_load(From, _Options)|T]) -->
 1006    [ nl, '\t~p'-[From] ],
 1007    sources(T)