/* Part of ClioPatria Author: Jan Wielemaker E-mail: J.Wielemaker@vu.nl WWW: http://cliopatria.swi-prolog.org Copyright (C): 2004-2017, University of Amsterdam VU University Amsterdam This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU Lesser General Public License along with this library; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA As a special exception, if you link this library with other files, compiled with a Free Software compiler, to produce an executable, this library does not by itself cause the resulting executable to be covered by the GNU General Public License. This exception does not however invalidate any other reasons why the executable file might be covered by the GNU General Public License. */ :- module(cp_server, [ cp_server/0, cp_server/1, % +Options cp_welcome/0, cp_after_load/1 % :Goal ]). /** ClioPatria main module This module loads the ClioPatria server as a library, providing the public predicates defined in the header. Before loading this file, the user should set up a the search path =cliopatria=. For example: == :- dynamic user:file_search_path/2. :- multifile user:file_search_path/2. user:file_search_path(cliopatria, '/usr/local/cliopatria'). :- use_module(cliopatria(cliopatria)). == @see http://cliopatria.swi-prolog.org */ :- dynamic user:file_search_path/2. :- multifile user:file_search_path/2. :- ( user:file_search_path(cliopatria, _) -> true ; prolog_load_context(directory, Dir), assert(user:file_search_path(cliopatria, Dir)) ). user:file_search_path(library, cliopatria(lib)). :- use_module(library(version)). :- check_prolog_version(or(70600, 70514)). % Demand >= 7.6.0, 7.5.14 :- register_git_module('ClioPatria', [ home_url('http://cliopatria.swi-prolog.org/') ]). :- use_module([ parms, skin(cliopatria), % HTML Page layout library(option), library(bundle), library(debug), library(lists), library(settings), library(error), library(broadcast), library(thread_pool), library(apply), library(semweb/rdf_db), library(semweb/rdf_persistency), library(semweb/rdf_litindex), library(semweb/rdf_ntriples), library(http/http_session), library(http/http_server_files), library(http/http_dispatch), library(http/thread_httpd), user(user_db), user(openid), user(preferences), api(sesame), api(journal), % export journal information api(sparql), api(export), api(void), applications(admin), applications(user), applications(browse), applications(yasgui), library(conf_d), user:library(cpack/cpack) ]). :- http_handler(web(.), serve_files_in_directory(web), [prefix]). :- dynamic after_load_goal/1. %% cp_server is det. %% cp_server(:Options) is det. % % Start the HTTP server. This predicate preforms the following % steps: % % 1. Load application settings from =|settings.db|= % 2. Load user-data from =|users.db|= % 3. Start the HTTP server % 4. Load the RDF persistent database from =|RDF-store|= % 5. Execute `after load' options registered using % cp_after_load/1. % % Defined options are: % % * port(Port) % Attach to Port instead of the port specified in the % configuration file settings.db. % * workers(+Count) % Number of worker threads to use. Default is the setting % =|http:workers|= % * prefix(+Prefix) % Rebase the server. See also the setting =|http:prefix|=. % * store(+Store) % Directory to use as persistent store. See also the % setting =|cliopatria:persistent_store|=. % * settings(+Settings) % Settings file. Default is =settings.db=. :- meta_predicate cp_server(:). cp_server :- current_prolog_flag(argv, [cpack|Argv]), !, load_conf_d([ 'config-enabled' ], []), cpack_control(Argv). :- if(current_predicate(http_unix_daemon:http_daemon/0)). cp_server :- http_unix_daemon:http_daemon. :- else. cp_server :- process_argv(Options, PrologFiles, RDFInputs), load_application(Options), user:maplist(ensure_loaded, PrologFiles), catch(cp_server([rdf_load(RDFInputs)|Options]), E, true), ( var(E) -> set_prolog_flag(toplevel_goal, prolog) % become interactive ; print_message(error, E), ( E = error(socket_error('Address already in use'), _) -> print_message(error, cliopatria(use_port_option)) ; true ) ). :- endif. cp_server(_Options) :- setting(http:port, DefPort), http_server_property(DefPort, goal(cp_server:http_dispatch)), !, print_message(informational, cliopatria(server_already_running(DefPort))). cp_server(Options) :- meta_options(is_meta, Options, QOptions), load_application(QOptions), option(settings(SettingsFile), QOptions, 'settings.db'), load_settings(SettingsFile), set_prefix(QOptions), attach_account_info, set_session_options, create_log_directory, setting(http:port, DefPort), setting(http:workers, DefWorkers), setting(http:worker_options, Settings), https_options(HTTPSOptions), merge_options(QOptions, Settings, HTTPOptions0), merge_options(HTTPOptions0, HTTPSOptions, HTTPOptions), option(port(Port), QOptions, DefPort), update_public_port(Port, DefPort), option(workers(Workers), QOptions, DefWorkers), http_server(http_dispatch, [ port(Port), workers(Workers) | HTTPOptions ]), option(after_load(AfterLoad), QOptions, true), option(rdf_load(RDFInputs), QOptions, []), print_message(informational, cliopatria(server_started(Port))), setup_call_cleanup( http_handler(root(.), busy_loading, [ priority(1000), hide_children(true), id(busy_loading), prefix ]), rdf_attach_store(QOptions, after_load(AfterLoad, RDFInputs)), http_delete_handler(id(busy_loading))). is_meta(after_load). :- public after_load/2. :- meta_predicate after_load(0, +). after_load(AfterLoad, RDFInputs) :- forall(member(Input, RDFInputs), call_warn(rdf_load(Input))), call(AfterLoad). set_prefix(Options) :- option(prefix(Prefix), Options), \+ setting(http:prefix, Prefix), !, set_setting_default(http:prefix, Prefix). set_prefix(_). %% update_public_port(+Port, +DefPort) % % Update http:public_port if port is changed using --port=Port. % Without this hack it is no longer to login after using the port % option. update_public_port(Port, Port) :- !. update_public_port(Port, DefPort) :- setting(http:public_port, DefPort), !, set_setting_default(http:public_port, Port), assertion(setting(http:public_port, Port)). update_public_port(_, _). %% load_application(+Options) % % Load cpack and local configuration. :- dynamic application_loaded/0. :- volatile application_loaded/0. load_application(_Options) :- application_loaded, !. load_application(_Options) :- load_conf_d([ cliopatria('config-enabled'), 'config-enabled' ], []), load_local, assertz(application_loaded). load_local :- absolute_file_name(local, Local, [ file_type(prolog), access(read), file_errors(fail) ]), !, print_message(informational, conf_d(load(Local))), ensure_loaded(user:Local). load_local. %% rdf_attach_store(+Options, :AfterLoad) is det. % % Attach the RDF store using the setting % cliopatria:persistent_store and call the `after-load' goals. % % @see cp_after_load/1 for registering after-load goals. :- meta_predicate rdf_attach_store(+, 0), call_warn(0). rdf_attach_store(Options, AfterLoad) :- ( option(store(Directory), Options) -> true ; setting(cliopatria:persistent_store, Directory) ), setup_indices, ( Directory \== '' -> rdf_attach_db(Directory, Options) ; true ), forall(after_load_goal(Goal), call_warn(Goal)), call_warn(AfterLoad). call_warn(Goal) :- ( catch(Goal, E, true) -> ( var(E) -> true ; print_message(warning, E) ) ; print_message(warning, goal_failed(Goal)) ). %% setup_indices is det. % % Initialize maintenance of the full-text indices. These indices % are created on first call and maintained dynamically as the RDF % store changes. By initializing them before there is any RDF % loaded, they will be built while the data is (re-)loaded, which % avoids long delays on the first query. Note that most of the % work is done in a separate thread. setup_indices :- setting(cliopatria:pre_index_tokens, true), rdf_find_literals(not_a_token, _), fail. setup_indices :- setting(cliopatria:pre_index_stems, true), rdf_find_literals(stem(not_a_stem), _), fail. setup_indices. %% cp_after_load(:Goal) is det. % % Register Goal to be executed after reloading the RDF persistent % DB. Note that already registered goals are not duplicated. % Running a goal after loading the database is commonly used to % ensure presence of relevant schemas or build additional indices. % Note that it is possible to start a thread for time-consuming % tasks (see thread_create/3). :- meta_predicate cp_after_load(0). cp_after_load(Goal) :- ( after_load_goal(Goal) -> true ; assert(after_load_goal(Goal)) ). %% busy_loading(+Request) % % This HTTP handler is pushed to overrule all actions of the % server while the server is restoring its persistent state. It % replies with the 503 (unavailable) response, indicating the % progress of restoring the repository. :- dynamic loading_done/2. busy_loading(_Request) :- rdf_statistics(triples(Triples)), ( loading_done(Nth, Total) -> Extra = [ '; ~D of ~D graphs.'-[Nth, Total] ] ; Extra = [ '.' ] ), HTML = p([ 'This service is currently restoring its ', 'persistent database.', br([]), 'Loaded ~D triples'-[Triples] | Extra ]), throw(http_reply(unavailable(HTML))). %% attach_account_info % % Set the registered user-database from the setting % cliopatria:user_data. attach_account_info :- setting(cliopatria:user_data, File), set_user_database(File). %% set_session_options % % Initialise session timeout from =|http:max_idle_time|=. set_session_options :- setting(http:max_idle_time, Idle), http_set_session_options([timeout(Idle)]). %% create_log_directory % % Create the directory in which the log files reside. create_log_directory :- current_setting(http:logfile), setting(http:logfile, File), File \== '', file_directory_name(File, DirName), DirName \== '.', !, catch(make_directory_path(DirName), E, print_message(warning, E)). create_log_directory. /******************************* * UPDATE SETTINGS * *******************************/ update_workers(New) :- setting(http:port, Port), http_current_worker(Port, _), http_workers(Port, New). :- listen(settings(changed(http:max_idle_time, _, New)), http_set_session_options([timeout(New)])). :- listen(settings(changed(http:workers, _, New)), update_workers(New)). /******************************* * ARGV * *******************************/ %% process_argv(-Options, -PrologFiles, -RDFInputs) % % Processes the ClioPatria commandline options. process_argv(Options, PrologFiles, RDFInputs) :- current_prolog_flag(argv, Argv), current_prolog_flag(os_argv, [Program|_]), ( Argv == ['--help'] -> usage(Program) ; catch(( parse_options(Argv, Options, Rest), maplist(load_argument, Rest, Load), keysort(Load, Sorted), group_pairs_by_key(Sorted, Keyed), ( memberchk(prolog-PrologFiles, Keyed) -> true ; PrologFiles = [] ), ( memberchk(rdf-RDFInputs, Keyed) -> true ; RDFInputs = [] ) ), E, ( print_message(error, E), fail )) -> true ; usage(Program) ). load_argument(URL, rdf-URL) :- ( sub_atom('http://', 0, _, _, URL) ; sub_atom('https://', 0, _, _, URL) ), !. load_argument(File, Type-File) :- file_name_extension(_Base, Ext, File), load_argument(Ext, File, Type). load_argument(Ext, _File, prolog) :- user:prolog_file_type(Ext, prolog), !. load_argument(gz, File, rdf) :- file_name_extension(Plain, gz, File), file_name_extension(_, RDF, Plain), rdf_extension(RDF). load_argument(RDF, _File, rdf) :- rdf_extension(RDF). rdf_extension(rdf). rdf_extension(owl). rdf_extension(ttl). rdf_extension(nt). rdf_extension(ntriples). cmd_option(-, help, -, 'Print command usage'). cmd_option(p, port, positive_integer, 'Port to connect to'). cmd_option(w, workers, positive_integer, 'Number of workers to start'). cmd_option(-, after_load, term, 'Goal to run after loading'). cmd_option(-, prefix, atom, 'Rebase the server to prefix/'). cmd_option(-, store, atom, 'Directory for persistent store'). % dummy to stop list_trivial_fail from warning about long_option/2. cmd_option(-, -, boolean, 'Dummy') :- fail. usage(Program) :- format(user_error, 'Run ClioPatria for interactive usage.~n~n', []), ansi_format([bold], 'Usage: ~w [options] arguments', [Program]), nl, nl, flush_output, forall(cmd_option(Short, Long, Type, Comment), describe_option(Short, Long, Type, Comment)), cpack_usage(Program), describe_argv, ( current_prolog_flag(hwnd, _) % swipl-win.exe console -> ansi_format([bold,hfg(red)], '~nPress \'b\' for break, any other key to exit > ', []), get_single_char(Key), ( Key == 0'b -> nl, nl, break ; true ), halt ; halt(1) ). describe_option(-, Long, -, Comment) :- !, format(user_error, ' --~w~t~40|~w~n', [Long, Comment]). describe_option(-, Long, _, Comment) :- !, format(user_error, ' --~w=~w~t~40|~w~n', [Long, Long, Comment]). describe_option(Short, Long, -, Comment) :- !, format(user_error, ' -~w, --~w~t~40|~w~n', [Short, Long, Comment]). describe_option(Short, Long, _, Comment) :- !, format(user_error, ' -~w ~w, --~w=~w~t~40|~w~n', [Short, Long, Long, Long, Comment]). describe_argv :- current_prolog_flag(argv, Argv), ( Argv == ['--help'] -> true ; ansi_format([fg(red)], 'Program argv: ~q~n', [Argv]) ). cpack_usage(Program) :- nl, ansi_format([bold], 'CPACK commands', []), nl, nl, flush_output, format(user_error, ' ~w cpack install pack ...~n', [Program]), format(user_error, ' ~w cpack upgrade pack ...~n', [Program]), format(user_error, ' ~w cpack configure pack ...~n', [Program]). parse_options([], [], []). parse_options([--|Rest], [], Rest) :- !. parse_options([H|T], [Opt|OT], Rest) :- sub_atom(H, 0, _, _, --), !, ( sub_atom(H, B, _, A, =) -> B2 is B - 2, sub_atom(H, 2, B2, _, Name), sub_atom(H, _, A, 0, Value), long_option(Name, Value, Opt) ; sub_atom(H, 2, _, 0, Name), long_option(Name, Opt) ), parse_options(T, OT, Rest). parse_options([H|T], Opts, Rest) :- atom_chars(H, [-|Opts]), !, short_options(Opts, T, Opts, Rest). parse_options(Rest, [], Rest). short_options([], Av, Opts, Rest) :- parse_options(Av, Opts, Rest). short_options([H|T], Av, [Opt|OptT], Rest) :- cmd_option(H, Name, Type, _), ( Type == (-) -> Opt =.. [Name,true], short_options(T, Av, OptT, Rest) ; Av = [Av0|AvT], text_to_value(Type, Av0, Value), Opt =.. [Name,Value], short_options(T, AvT, OptT, Rest) ). long_option(Name, Text, Opt) :- cmd_option(_, Name, Type, _), text_to_value(Type, Text, Value), Opt =.. [Name,Value]. long_option(Name, Opt) :- atom_concat('no-', OptName, Name), cmd_option(_, OptName, boolean, _), !, Opt =.. [Name,false]. long_option(Name, Opt) :- cmd_option(_, Name, boolean, _), Opt =.. [Name,true]. text_to_value(boolean, Text, Value) :- downcase_atom(Text, Lwr), boolean(Lwr, Value). text_to_value(atom, Text, Text). text_to_value(oneof(L), Text, Text) :- memberchk(Text, L). text_to_value(integer, Text, Int) :- atom_number(Text, Int), integer(Int). text_to_value(nonneg, Text, Int) :- atom_number(Text, Int), integer(Int), Int >= 0. text_to_value(positive_integer, Text, Int) :- atom_number(Text, Int), integer(Int), Int > 0. text_to_value(negative_integer, Text, Int) :- atom_number(Text, Int), integer(Int), Int < 0. text_to_value(float, Text, Float) :- atom_number(Text, Number), Float = float(Number). text_to_value(term, Text, Term) :- atom_to_term(Text, Term, _). boolean(true, true). boolean(yes, true). boolean(on, true). boolean(false, false). boolean(no, false). boolean(off, false). /******************************* * CPACK * *******************************/ %% cpack_control(+Commands:list) % % Execute a CPACK configuration instruction. For example: % % ./run.pl cpack install swish cpack_control([install|Packs]) :- !, maplist(cpack_install, Packs). cpack_control([configure|Packs]) :- !, maplist(cpack_configure, Packs). cpack_control([upgrade|Packs]) :- !, ( Packs == [] -> cpack_upgrade ; maplist(cpack_upgrade, Packs) ). cpack_control(Command) :- domain_error(cpack_command, Command). /******************************* * BANNER * *******************************/ %% cp_welcome % % Print welcome banner. cp_welcome :- setting(http:port, Port), print_message(informational, cliopatria(welcome(Port))). /******************************* * POOLS * *******************************/ :- multifile http:create_pool/1. :- setting(cliopatria:max_clients, integer, 50, 'Max number of concurrent requests in ClioPatria pool'). :- if(current_prolog_flag(address_bits, 32)). :- setting(cliopatria:stack_size, integer, 128, 'Stack limit in MB for ClioPatria pool'). :- else. :- setting(cliopatria:stack_size, integer, 1024, 'Stack limit in MB for ClioPatria pool'). :- endif. %% http:create_pool(+Pool) is semidet. % % Create a thread-pool on-demand. http:create_pool(sparql_query) :- debug(http(pool), 'Demand-creating pool ~q', [sparql_query]), setting(sparql:max_clients, Count), setting(sparql:stack_size, MB), Global is MB * 1024, Trail is MB * 1024, thread_pool_create(sparql_query, Count, [ global(Global), trail(Trail) ]). http:create_pool(cliopatria) :- setting(cliopatria:max_clients, Count), setting(cliopatria:stack_size, MB), Global is MB * 1024, Trail is MB * 1024, thread_pool_create(cliopatria, Count, [ global(Global), trail(Trail) ]). /******************************* * HTTPS * *******************************/ %% https_options(-Options) is det. % % Fetch options for running an HTTPS server. HTTP is started if % there is a directory =https= with these files: % % $ =|server-cert.pem|= : % Contains the server certificate. This may be omitted, in % which case the =|server-key.pem|= is also passed using the % key_file(+File) option. % $ =|server-key.pem|= : % Contains the private key for the server. % % =|passwd|= : % Needs to hold the password if the private key is protected % with a password. https_options(Options) :- https_file('server-key.pem', KeyFile), !, ( https_file('server-cert.pem', CertFile) -> true ; CertFile = KeyFile ), Options = [ ssl([ certificate_file(CertFile), key_file(KeyFile) | PasswdOption ]) ], ( https_file(passwd, PasswordFile) -> read_file_to_string(PasswordFile, Content, []), split_string(Content, "", " \n\r", [Passwd]), PasswdOption = [password(Passwd)] ; PasswdOption = [] ). https_options([]). https_file(Base, File) :- absolute_file_name(config_https(Base), File, [ access(read), file_errors(fail) ]). /******************************* * MESSAGES * *******************************/ :- multifile prolog:message//1. prolog:message(cliopatria(server_started(_Port))) --> []. prolog:message(cliopatria(welcome(DefaultPort))) --> [ nl, 'Use one of the calls below to start the ClioPatria server:', nl, nl, ' ?- cp_server. % start at port ~w'-[DefaultPort], nl, ' ?- cp_server([port(Port)]). % start at Port' ]. prolog:message(cliopatria(use_port_option)) --> [ ' Could not start the HTTP server!', nl, ' Choose a different port using ./run.pl --port= or', nl, ' use the network plugin to change the default port.' ]. prolog:message(cliopatria(server_already_running(Port))) --> { cp_host(Port, Host), cp_port(Port, PublicPort), http_location_by_id(root, Root) }, [ 'CliopPatria server is already running at http://~w:~w~w'- [Host, PublicPort, Root] ]. cp_host(_, Host) :- setting(http:public_host, Host), Host \== '', !. cp_host(Host:_, Host) :- !. cp_host(_,Host) :- gethostname(Host). cp_port(_ServerPort, PublicPort) :- setting(http:public_host, Host), Host \== '', Host \== localhost, setting(http:public_port, PublicPort), !. cp_port(_Host:Port, Port) :- !. cp_port(ServerPort, ServerPort). /******************************* * HOOKS * *******************************/ :- multifile user:message_hook/3. user:message_hook(rdf(restore(_, done(_DB, _T, _Count, Nth, Total))), _Kind, _Lines) :- retractall(loading_done(_,_)), assert(loading_done(Nth, Total)), fail. :- multifile http_unix_daemon:http_server_hook/1. % +Options http_unix_daemon:http_server_hook(Options) :- cp_server(Options).