1/*  Part of ClioPatria SeRQL and SPARQL server
    2
    3    Author:        Jan Wielemaker
    4    E-mail:        J.Wielemaker@cs.vu.nl
    5    WWW:           http://cliopatria.swi-prolog.org
    6    Copyright (C): 2010-2017, University of Amsterdam,
    7			      VU University Amsterdam
    8
    9    This program is free software; you can redistribute it and/or
   10    modify it under the terms of the GNU General Public License
   11    as published by the Free Software Foundation; either version 2
   12    of the License, or (at your option) any later version.
   13
   14    This program is distributed in the hope that it will be useful,
   15    but WITHOUT ANY WARRANTY; without even the implied warranty of
   16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   17    GNU General Public License for more details.
   18
   19    You should have received a copy of the GNU General Public
   20    License along with this library; if not, write to the Free Software
   21    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   22
   23    As a special exception, if you link this library with other files,
   24    compiled with a Free Software compiler, to produce an executable, this
   25    library does not by itself cause the resulting executable to be covered
   26    by the GNU General Public License. This exception does not however
   27    invalidate any other reasons why the executable file might be covered by
   28    the GNU General Public License.
   29*/
   30
   31:- module(cpack_xref,
   32	  [ xref_cpack/1,
   33	    xref_cpack_file/1
   34	  ]).   35:- use_module(library(apply)).   36:- use_module(library(debug)).   37:- use_module(library(semweb/rdf_db)).   38:- use_module(library(semweb/rdfs)).   39:- use_module(library(git)).   40:- use_module(library(option)).   41:- use_module(library(uri)).   42:- use_module(library(prolog_xref)).   43:- use_module(library(prolog_source)).   44:- use_module(repository).   45
   46/** <module> Compute dependencies between CPACKs
   47
   48This module runs the Prolog  cross-referencer   on  a  submitted pack to
   49analyse the package dependencies.
   50*/
   51
   52%%	xref_cpack(+Pack) is det.
   53%
   54%	Create cross-reference info for a complete pack.
   55
   56xref_cpack(Pack) :-
   57	debug(xref),
   58	findall(File, pack_prolog_file(Pack, File), Files),
   59	maplist(xref_cpack_file, Files),
   60	maplist(resolve_file, Files).
   61
   62pack_prolog_file(Pack, File) :-
   63	rdf_has(File, cpack:inPack, Pack),
   64	rdf_has(File, cpack:name, literal(Name)),
   65	\+ rdf_has(File, cpack:ignored, literal(type(xsd:boolean, true))),
   66	file_name_extension(_, Ext, Name),
   67	prolog_file_type(Ext, prolog).
   68
   69
   70%%	xref_cpack_file(+File) is det.
   71%
   72%	Do cross-reference analysis on the CPACK   file  File. This adds
   73%	various properties to help dependency tracking to File:
   74%
   75%	  * cpack:module
   76%	  * cpack:exportsPredicate
   77
   78:- thread_local
   79	xref_git/0.   80
   81xref_cpack_file(File) :-
   82	print_message(informational, cpack(xref(File))),
   83	setup_call_cleanup(assert(xref_git, Ref),
   84			   (   xref_source(File),
   85			       xref_to_rdf(File)
   86			   ),
   87			   erase(Ref)).
   88
   89:- rdf_meta
   90	file_property(r,r,o).   91
   92xref_to_rdf(File) :-
   93	rdf_has(File, cpack:inPack, Graph),
   94	forall(setof(O, file_property(File, P, O), OL),
   95	       assert_objects(OL, File, P, Graph)).
   96
   97assert_objects([], _, _, _).
   98assert_objects([O|T], S, P, G) :-
   99	rdf_assert(S,P,O,G),
  100	assert_objects(T, S, P, G).
  101
  102
  103%%	file_property(+File, ?P, ?O) is nondet.
  104%
  105%	True when rdf(File,P,O) describes a property   of  File. Used to
  106%	generate all properties from the cross-referencer output.
  107%
  108%	@tbd	We can compute other dependencies (referenced RDF
  109%		namespaces, used HTML resources, etc.)
  110%	@tbd	We can also compute possible dangerous code.  See
  111%		safecode as developed as part of SWAPP.
  112
  113file_property(File, cpack:module, literal(Module)) :-
  114	xref_module(File, Module).
  115file_property(File, cpack:exportsPredicate, literal(Pred)) :-
  116	xref_exported(File, Callable),
  117	head_atom(Callable, Pred).
  118file_property(File, cpack:requiresPredicate, literal(Pred)) :-
  119	xref_called(File, Callable, _),
  120	\+ xref_local_defined(File, Callable),
  121	head_atom(Callable, Pred).
  122file_property(File, cpack:publicPredicate, literal(Pred)) :-
  123	xref_defined(File, Callable, public(_Line)),
  124	head_atom(Callable, Pred).
  125file_property(File, UsesFile, Uses) :-
  126	rdf_has(File, cpack:inPack, Pack),
  127	xref_uses_file(File, Spec, Path),
  128	(   rdf_is_resource(Path),
  129	    rdfs_individual_of(Path, cpack:'File'),
  130	    rdf_has(Path, cpack:inPack, Pack)
  131	->  rdf_equal(UsesFile, cpack:usesPackageFile),
  132	    Uses = Path
  133	;   file_ref(Spec, Uses),
  134	    (   Path == '<not_found>'
  135	    ->  rdf_equal(UsesFile, cpack:usesPackageFile)
  136	    ;   system_file(Path, FileURI, Graph)
  137	    ->  rdf_equal(UsesFile, cpack:usesSystemFile),
  138		rdf_assert(FileURI, cpack:resolves, Uses, Graph)
  139	    ;   cliopatria_file(Path, FileURI, Graph)
  140	    ->  rdf_equal(UsesFile, cpack:usesClioPatriaFile),
  141		rdf_assert(FileURI, cpack:resolves, Uses, Graph)
  142	    ;   rdf_equal(UsesFile, cpack:usesPackageFile)
  143	    )
  144	).
  145
  146
  147head_atom(Head, Atom) :-
  148	head_pi(Head, PI),
  149	format(atom(Atom), '~q', [PI]).
  150
  151head_pi(M:Term, M:PI) :- !,
  152	head_pi(Term, PI).
  153head_pi(Term, Name/Arity) :-
  154	functor(Term, Name, Arity).
  155
  156xref_local_defined(Src, Callable) :-
  157	xref_defined(Src, Callable, How),
  158	How \= imported(_From).
  159
  160
  161%%	system_file(+Path, -FileURI, -Graph) is semidet.
  162%%	cliopatria_file(+Path, -FileURI, -Graph) is semidet.
  163%
  164%	Classify file according to their origin.
  165
  166:- rdf_meta
  167	system_file_uri(+, +, r, r, r).  168
  169system_file(Path, FileURI, Graph) :-
  170	current_prolog_flag(home, Home),
  171	sub_atom(Path, 0, _, _, Home), !,
  172	cpack_uri(graph, prolog, Graph),
  173	system_file_uri(Path, Home, cpack:'SystemFile', Graph, FileURI).
  174
  175cliopatria_file(Path, FileURI, Graph) :-
  176	absolute_file_name(cliopatria(.),
  177			   Home,
  178			   [ file_type(directory),
  179			     access(read)
  180			   ]),
  181	sub_atom(Path, 0, _, _, Home),
  182	\+ loaded_package_file(Path), !,
  183	cpack_uri(graph, cliopatria, Graph),
  184	system_file_uri(Path, Home, cpack:'ClioPatriaFile', Graph, FileURI).
  185
  186loaded_package_file(Path) :-
  187	setting(cpack:package_directory, PackageDir),
  188	absolute_file_name(PackageDir, PackageRoot),
  189	sub_atom(Path, 0, _, _, PackageRoot).
  190
  191system_file_uri(Path, Root, Class, Graph, URI) :-
  192	directory_file_path(Root, RelPath, Path),
  193	cpack_uri(prolog, RelPath, URI),
  194	(   rdfs_individual_of(URI, cpack:'File')
  195	->  update_exports(Path, URI, Graph)
  196	;   file_base(RelPath, Base),
  197	    file_base_name(RelPath, FileName),
  198	    rdf_assert(URI, rdf:type, Class, Graph),
  199	    rdf_assert(URI, cpack:path, literal(RelPath), Graph),
  200	    rdf_assert(URI, cpack:name, literal(FileName), Graph),
  201	    rdf_assert(URI, cpack:base, literal(Base), Graph),
  202	    update_exports(Path, URI, Graph)
  203	).
  204
  205update_exports(Path, URI, Graph) :-
  206	time_file(Path, Time),
  207	format_time(atom(XSD), '%FT%T%:z', Time),
  208	rdf_equal(xsd:dateTime, Type),
  209	RDFStamp = literal(type(Type, XSD)),
  210	(   rdf_has(URI, cpack:lastModifiedTime, RDFStamp)
  211	->  true
  212	;   rdf_retractall(URI, cpack:lastModifiedTime, _,  Graph),
  213	    rdf_retractall(URI, cpack:exportsPredicate, _, Graph),
  214	    rdf_retractall(URI, cpack:publicPredicate, _, Graph),
  215	    rdf_retractall(URI, cpack:module, _, Graph),
  216	    rdf_assert(URI, cpack:lastModifiedTime, RDFStamp, Graph),
  217	    assert_exports(Path, URI, Graph)
  218	).
  219
  220assert_exports(Path, URI, Graph) :-
  221	xref_public_list(Path, _, Module, Exports, Public, _Meta, -), !,
  222	rdf_assert(URI, cpack:module, literal(Module), Graph),
  223	forall(member(PI, Exports),
  224	       (   cannonical_pi(PI, CannPI),
  225		   format(atom(Id), '~q', [CannPI]),
  226		   rdf_assert(URI, cpack:exportsPredicate,
  227			      literal(Id), Graph)
  228	       )),
  229	forall(member(PI, Public),
  230	       (   cannonical_pi(Module:PI, CannPI),
  231		   format(atom(Id), '~q', [CannPI]),
  232		   rdf_assert(URI, cpack:publicPredicate,
  233			      literal(Id), Graph)
  234	       )).
  235assert_exports(_,_,_).
  236
  237
  238cannonical_pi(M:PI, M:CannPi) :- !,
  239	cannonical_pi(PI, CannPi).
  240cannonical_pi(Name//DCGArity, Name/Arity) :- !,
  241	Arity is DCGArity + 2.
  242cannonical_pi(PI, PI).
  243
  244		 /*******************************
  245		 *	   FIND FILES		*
  246		 *******************************/
  247
  248%%	search_file(+Spec, -File:uri) is nondet.
  249%
  250%	True when File is a candidate   for resolving the given symbolic
  251%	path Spec. The trick is that  the   path  =cpacks= refers to all
  252%	packs in our database, so we   need to rewrite our specification
  253%	as cpacks(Path).  I.e.,
  254%
  255%		library(X) --> cpacks(lib/X)
  256%
  257%	The last clause finds embedded Prolog packs, assuming these have
  258%	a meta-file `pack.pl` and a directory  `prolog` that is attached
  259%	to the library path.
  260
  261search_file(Spec, File) :-	% find applications(...), icons(...), ...
  262	path_rule(Spec, cpacks(Segments)),
  263	path_segments_atom(Segments, InPack),
  264	add_pl_ext(InPack, Target),
  265	once(rdf_has(File, cpack:path, literal(Target))).
  266search_file(Spec, File) :-	% find Package(...)
  267	rdf_has(_Pack, cpack:packageName, literal(PackName)),
  268	Spec =.. [PackName,Segments],
  269	path_segments_atom(Segments, InPack),
  270	add_pl_ext(InPack, Target),
  271	once(rdf_has(File, cpack:path, literal(Target))).
  272search_file(library(Segments), File) :-
  273	path_segments_atom(Segments, Local),
  274	add_pl_ext(Local, LocalPL),
  275	rdf_has(Meta, cpack:name, literal('pack.pl')),
  276	directory_file_path(prolog, LocalPL, LibFile),
  277	uri_normalized(LibFile, Meta, File),
  278	rdf(File, rdf:type, cpack:'PrologFile').
  279
  280add_pl_ext(Base, Base).
  281add_pl_ext(Base, BasePL) :-
  282	user:prolog_file_type(Ext, prolog),
  283	file_name_extension(Base, Ext, BasePL).
  284
  285path_rule(Alias, NewAlias) :-
  286	Alias =.. [Name,Local],
  287	user:file_search_path(Name, Exp),
  288	Exp =.. [NewName,Parent],
  289	NewAlias =.. [NewName,Parent/Local].
  290
  291
  292		 /*******************************
  293		 *	 FILE REFERENCES	*
  294		 *******************************/
  295
  296%%	file_ref(+Spec, -URI) is det.
  297%
  298%	True when URI is the URI for  a FileRef instance that represents
  299%	the  symbolic  file-reference  Spec.  If  there  is  already  an
  300%	existing URI for the file-reference, this is returned. Otherwise
  301%	it creates an instance of cpack:FileRef.
  302
  303file_ref(Spec, URI) :-
  304	must_be(ground, Spec),
  305	format(atom(Id), '~q', [Spec]),
  306	cpack_uri(file_ref, Id, URI),
  307	(   rdf(URI, rdf:type, cpack:'FileRef')
  308	->  true
  309	;   cpack_uri(graph, 'file-references', Graph),
  310	    file_base(Spec, BaseName),
  311	    rdf_assert(URI, rdf:type, cpack:'FileRef', Graph),
  312	    rdf_assert(URI, cpack:name, literal(Id), Graph),
  313	    rdf_assert(URI, cpack:base, literal(BaseName), Graph),
  314	    resolve_file_ref(URI)
  315	).
  316
  317file_base(Spec, Base) :-
  318	atom(Spec), !,
  319	file_base_name(Spec, File),
  320	file_name_extension(Base, _Ext, File). % demand Prolog?
  321file_base(_/Rest, Base) :- !,
  322	file_base(Rest, Base).
  323file_base(Spec, Base) :-
  324	arg(1, Spec, Name),
  325	file_base(Name, Base).
  326
  327%%	resolve_file(+File) is det.
  328%
  329%	Create links to the  FileRef  objects   to  which  this file may
  330%	resolve.
  331
  332resolve_file(File) :-
  333	rdf_has(File, cpack:base, Base),
  334	findall(FileRef,
  335		(   rdf_has(FileRef, cpack:base, Base),
  336		    rdfs_individual_of(FileRef, cpack:'FileRef')
  337		),
  338		Candidates),
  339	maplist(resolve_file_ref, Candidates).
  340
  341
  342%%	resolve_file_ref(+FileRef) is det.
  343%
  344%	File files to which FileRef can resolve  and add relate the file
  345%	to this FileRef.
  346
  347resolve_file_ref(FileRef) :-
  348	rdf_has(FileRef, cpack:name, literal(Id)),
  349	atom_to_term(Id, Spec, _Vars),
  350	forall(search_file(Spec, File),
  351	       assert_resolves(File, FileRef)).
  352
  353assert_resolves(File, FileRef) :-
  354	rdf_has(File, cpack:resolves, FileRef), !.
  355assert_resolves(File, FileRef) :-
  356	rdf_has(File, cpack:inPack, Pack),
  357	rdf_assert(File, cpack:resolves, FileRef, Pack).
  358
  359
  360		 /*******************************
  361		 *   HOOKS TO WORK ON GIT REPO	*
  362		 *******************************/
  363
  364:- multifile
  365	prolog:xref_open_source/2,
  366	prolog:xref_source_identifier/2,
  367	prolog:xref_source_file/3.  368
  369prolog:xref_open_source(File, Stream) :-
  370	rdf_is_resource(File),
  371	rdf_has(File, cpack:path, literal(Path)),
  372	rdf_has(File, cpack:inPack, Pack),
  373	(   rdf_has(File, cpack:mirrorRepository, Mirror),
  374	    rdf_has(Mirror, cpack:branch, literal(Branch))
  375	->  true
  376	;   Branch = master
  377	),
  378	cpack_our_mirror(Pack, BareGitDir),
  379	git_open_file(BareGitDir, Path, Branch, Stream).
  380
  381prolog:xref_source_identifier(File, File) :-
  382	rdf_is_resource(File),
  383	rdfs_individual_of(File, cpack:'PrologFile').
  384
  385%%	prolog:xref_source_file(+Spec, -File, +Options) is semidet.
  386%
  387%	True when File is the URI  of   a  file referenced by Spec. This
  388%	predicate hooks into xref_source_file/3, making  it possible for
  389%	the cross-referencer to analyse files in GIT repositories.
  390
  391prolog:xref_source_file(Spec, File, _Options) :-
  392	xref_git,
  393	rdf_is_resource(Spec),
  394	(   File = Spec
  395	;   prolog_file_type(Ext, prolog),
  396	    file_name_extension(Spec, Ext, File)
  397	),
  398	rdfs_individual_of(File, cpack:'File'), !.
  399prolog:xref_source_file(Spec, File, Options) :-
  400	xref_git,
  401	atom(Spec),
  402	option(relative_to(Base), Options),
  403	uri_normalized(Spec, Base, Spec2),
  404	(   File = Spec2
  405	;   prolog_file_type(Ext, prolog),
  406	    file_name_extension(Spec2, Ext, File)
  407	),
  408	rdfs_individual_of(File, cpack:'File'), !.
  409prolog:xref_source_file(Spec, File, _Options) :-
  410	xref_git,
  411	search_file(Spec, File), !,
  412					% but NOT a ClioPatria file
  413	\+ ( path_rule(Spec, cpacks(Segments)),
  414	     path_segments_atom(Segments, InPack),
  415	     absolute_file_name(cliopatria(InPack), _,
  416				[ file_type(prolog),
  417				  access(read),
  418				  file_errors(fail)
  419				])).
  420
  421
  422		 /*******************************
  423		 *	      MESSAGES		*
  424		 *******************************/
  425
  426:- multifile
  427	prolog:message//1.  428
  429prolog:message(cpack(xref(File))) -->
  430	{ rdf_has(File, cpack:path, literal(Path)) },
  431	[ 'Analyzing ~w'-[Path] ]