View source with raw comments or as raw
    1/*  Part of ClioPatria SeRQL and SPARQL server
    2
    3    Author:        Jan Wielemaker
    4    E-mail:        J.Wielemaker@cs.vu.nl
    5    WWW:           http://cliopatria.swi-prolog.org
    6    Copyright (C): 2010-2017, University of Amsterdam,
    7			      VU University Amsterdam
    8
    9    This program is free software; you can redistribute it and/or
   10    modify it under the terms of the GNU General Public License
   11    as published by the Free Software Foundation; either version 2
   12    of the License, or (at your option) any later version.
   13
   14    This program is distributed in the hope that it will be useful,
   15    but WITHOUT ANY WARRANTY; without even the implied warranty of
   16    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   17    GNU General Public License for more details.
   18
   19    You should have received a copy of the GNU General Public
   20    License along with this library; if not, write to the Free Software
   21    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
   22
   23    As a special exception, if you link this library with other files,
   24    compiled with a Free Software compiler, to produce an executable, this
   25    library does not by itself cause the resulting executable to be covered
   26    by the GNU General Public License. This exception does not however
   27    invalidate any other reasons why the executable file might be covered by
   28    the GNU General Public License.
   29*/
   30
   31:- module(cpack_xref,
   32	  [ xref_cpack/1,
   33	    xref_cpack_file/1
   34	  ]).   35:- use_module(library(apply)).   36:- use_module(library(debug)).   37:- use_module(library(semweb/rdf_db)).   38:- use_module(library(semweb/rdfs)).   39:- use_module(library(git)).   40:- use_module(library(option)).   41:- use_module(library(uri)).   42:- use_module(library(prolog_xref)).   43:- use_module(library(prolog_source)).   44:- use_module(repository).

Compute dependencies between CPACKs

This module runs the Prolog cross-referencer on a submitted pack to analyse the package dependencies. */

 xref_cpack(+Pack) is det
Create cross-reference info for a complete pack.
   56xref_cpack(Pack) :-
   57	debug(xref),
   58	findall(File, pack_prolog_file(Pack, File), Files),
   59	maplist(xref_cpack_file, Files),
   60	maplist(resolve_file, Files).
   61
   62pack_prolog_file(Pack, File) :-
   63	rdf_has(File, cpack:inPack, Pack),
   64	rdf_has(File, cpack:name, literal(Name)),
   65	\+ rdf_has(File, cpack:ignored, literal(type(xsd:boolean, true))),
   66	file_name_extension(_, Ext, Name),
   67	prolog_file_type(Ext, prolog).
 xref_cpack_file(+File) is det
Do cross-reference analysis on the CPACK file File. This adds various properties to help dependency tracking to File:
   78:- thread_local
   79	xref_git/0.   80
   81xref_cpack_file(File) :-
   82	print_message(informational, cpack(xref(File))),
   83	setup_call_cleanup(assert(xref_git, Ref),
   84			   (   xref_source(File),
   85			       xref_to_rdf(File)
   86			   ),
   87			   erase(Ref)).
   88
   89:- rdf_meta
   90	file_property(r,r,o).   91
   92xref_to_rdf(File) :-
   93	rdf_has(File, cpack:inPack, Graph),
   94	forall(setof(O, file_property(File, P, O), OL),
   95	       assert_objects(OL, File, P, Graph)).
   96
   97assert_objects([], _, _, _).
   98assert_objects([O|T], S, P, G) :-
   99	rdf_assert(S,P,O,G),
  100	assert_objects(T, S, P, G).
 file_property(+File, ?P, ?O) is nondet
True when rdf(File,P,O) describes a property of File. Used to generate all properties from the cross-referencer output.
To be done
- We can compute other dependencies (referenced RDF namespaces, used HTML resources, etc.)
- We can also compute possible dangerous code. See safecode as developed as part of SWAPP.
  113file_property(File, cpack:module, literal(Module)) :-
  114	xref_module(File, Module).
  115file_property(File, cpack:exportsPredicate, literal(Pred)) :-
  116	xref_exported(File, Callable),
  117	head_atom(Callable, Pred).
  118file_property(File, cpack:requiresPredicate, literal(Pred)) :-
  119	xref_called(File, Callable, _),
  120	\+ xref_local_defined(File, Callable),
  121	head_atom(Callable, Pred).
  122file_property(File, cpack:publicPredicate, literal(Pred)) :-
  123	xref_defined(File, Callable, public(_Line)),
  124	head_atom(Callable, Pred).
  125file_property(File, UsesFile, Uses) :-
  126	rdf_has(File, cpack:inPack, Pack),
  127	xref_uses_file(File, Spec, Path),
  128	(   rdf_is_resource(Path),
  129	    rdfs_individual_of(Path, cpack:'File'),
  130	    rdf_has(Path, cpack:inPack, Pack)
  131	->  rdf_equal(UsesFile, cpack:usesPackageFile),
  132	    Uses = Path
  133	;   file_ref(Spec, Uses),
  134	    (   Path == '<not_found>'
  135	    ->  rdf_equal(UsesFile, cpack:usesPackageFile)
  136	    ;   system_file(Path, FileURI, Graph)
  137	    ->  rdf_equal(UsesFile, cpack:usesSystemFile),
  138		rdf_assert(FileURI, cpack:resolves, Uses, Graph)
  139	    ;   cliopatria_file(Path, FileURI, Graph)
  140	    ->  rdf_equal(UsesFile, cpack:usesClioPatriaFile),
  141		rdf_assert(FileURI, cpack:resolves, Uses, Graph)
  142	    ;   rdf_equal(UsesFile, cpack:usesPackageFile)
  143	    )
  144	).
  145
  146
  147head_atom(Head, Atom) :-
  148	head_pi(Head, PI),
  149	format(atom(Atom), '~q', [PI]).
  150
  151head_pi(M:Term, M:PI) :- !,
  152	head_pi(Term, PI).
  153head_pi(Term, Name/Arity) :-
  154	functor(Term, Name, Arity).
  155
  156xref_local_defined(Src, Callable) :-
  157	xref_defined(Src, Callable, How),
  158	How \= imported(_From).
 system_file(+Path, -FileURI, -Graph) is semidet
 cliopatria_file(+Path, -FileURI, -Graph) is semidet
Classify file according to their origin.
  166:- rdf_meta
  167	system_file_uri(+, +, r, r, r).  168
  169system_file(Path, FileURI, Graph) :-
  170	current_prolog_flag(home, Home),
  171	sub_atom(Path, 0, _, _, Home), !,
  172	cpack_uri(graph, prolog, Graph),
  173	system_file_uri(Path, Home, cpack:'SystemFile', Graph, FileURI).
  174
  175cliopatria_file(Path, FileURI, Graph) :-
  176	absolute_file_name(cliopatria(.),
  177			   Home,
  178			   [ file_type(directory),
  179			     access(read)
  180			   ]),
  181	sub_atom(Path, 0, _, _, Home),
  182	\+ loaded_package_file(Path), !,
  183	cpack_uri(graph, cliopatria, Graph),
  184	system_file_uri(Path, Home, cpack:'ClioPatriaFile', Graph, FileURI).
  185
  186loaded_package_file(Path) :-
  187	setting(cpack:package_directory, PackageDir),
  188	absolute_file_name(PackageDir, PackageRoot),
  189	sub_atom(Path, 0, _, _, PackageRoot).
  190
  191system_file_uri(Path, Root, Class, Graph, URI) :-
  192	directory_file_path(Root, RelPath, Path),
  193	cpack_uri(prolog, RelPath, URI),
  194	(   rdfs_individual_of(URI, cpack:'File')
  195	->  update_exports(Path, URI, Graph)
  196	;   file_base(RelPath, Base),
  197	    file_base_name(RelPath, FileName),
  198	    rdf_assert(URI, rdf:type, Class, Graph),
  199	    rdf_assert(URI, cpack:path, literal(RelPath), Graph),
  200	    rdf_assert(URI, cpack:name, literal(FileName), Graph),
  201	    rdf_assert(URI, cpack:base, literal(Base), Graph),
  202	    update_exports(Path, URI, Graph)
  203	).
  204
  205update_exports(Path, URI, Graph) :-
  206	time_file(Path, Time),
  207	format_time(atom(XSD), '%FT%T%:z', Time),
  208	rdf_equal(xsd:dateTime, Type),
  209	RDFStamp = literal(type(Type, XSD)),
  210	(   rdf_has(URI, cpack:lastModifiedTime, RDFStamp)
  211	->  true
  212	;   rdf_retractall(URI, cpack:lastModifiedTime, _,  Graph),
  213	    rdf_retractall(URI, cpack:exportsPredicate, _, Graph),
  214	    rdf_retractall(URI, cpack:publicPredicate, _, Graph),
  215	    rdf_retractall(URI, cpack:module, _, Graph),
  216	    rdf_assert(URI, cpack:lastModifiedTime, RDFStamp, Graph),
  217	    assert_exports(Path, URI, Graph)
  218	).
  219
  220assert_exports(Path, URI, Graph) :-
  221	xref_public_list(Path, _, Module, Exports, Public, _Meta, -), !,
  222	rdf_assert(URI, cpack:module, literal(Module), Graph),
  223	forall(member(PI, Exports),
  224	       (   cannonical_pi(PI, CannPI),
  225		   format(atom(Id), '~q', [CannPI]),
  226		   rdf_assert(URI, cpack:exportsPredicate,
  227			      literal(Id), Graph)
  228	       )),
  229	forall(member(PI, Public),
  230	       (   cannonical_pi(Module:PI, CannPI),
  231		   format(atom(Id), '~q', [CannPI]),
  232		   rdf_assert(URI, cpack:publicPredicate,
  233			      literal(Id), Graph)
  234	       )).
  235assert_exports(_,_,_).
  236
  237
  238cannonical_pi(M:PI, M:CannPi) :- !,
  239	cannonical_pi(PI, CannPi).
  240cannonical_pi(Name//DCGArity, Name/Arity) :- !,
  241	Arity is DCGArity + 2.
  242cannonical_pi(PI, PI).
  243
  244		 /*******************************
  245		 *	   FIND FILES		*
  246		 *******************************/
 search_file(+Spec, -File:uri) is nondet
True when File is a candidate for resolving the given symbolic path Spec. The trick is that the path cpacks refers to all packs in our database, so we need to rewrite our specification as cpacks(Path). I.e.,
library(X) --> cpacks(lib/X)

The last clause finds embedded Prolog packs, assuming these have a meta-file pack.pl and a directory prolog that is attached to the library path.

  261search_file(Spec, File) :-	% find applications(...), icons(...), ...
  262	path_rule(Spec, cpacks(Segments)),
  263	path_segments_atom(Segments, InPack),
  264	add_pl_ext(InPack, Target),
  265	once(rdf_has(File, cpack:path, literal(Target))).
  266search_file(Spec, File) :-	% find Package(...)
  267	rdf_has(_Pack, cpack:packageName, literal(PackName)),
  268	Spec =.. [PackName,Segments],
  269	path_segments_atom(Segments, InPack),
  270	add_pl_ext(InPack, Target),
  271	once(rdf_has(File, cpack:path, literal(Target))).
  272search_file(library(Segments), File) :-
  273	path_segments_atom(Segments, Local),
  274	add_pl_ext(Local, LocalPL),
  275	rdf_has(Meta, cpack:name, literal('pack.pl')),
  276	directory_file_path(prolog, LocalPL, LibFile),
  277	uri_normalized(LibFile, Meta, File),
  278	rdf(File, rdf:type, cpack:'PrologFile').
  279
  280add_pl_ext(Base, Base).
  281add_pl_ext(Base, BasePL) :-
  282	user:prolog_file_type(Ext, prolog),
  283	file_name_extension(Base, Ext, BasePL).
  284
  285path_rule(Alias, NewAlias) :-
  286	Alias =.. [Name,Local],
  287	user:file_search_path(Name, Exp),
  288	Exp =.. [NewName,Parent],
  289	NewAlias =.. [NewName,Parent/Local].
  290
  291
  292		 /*******************************
  293		 *	 FILE REFERENCES	*
  294		 *******************************/
 file_ref(+Spec, -URI) is det
True when URI is the URI for a FileRef instance that represents the symbolic file-reference Spec. If there is already an existing URI for the file-reference, this is returned. Otherwise it creates an instance of cpack:FileRef.
  303file_ref(Spec, URI) :-
  304	must_be(ground, Spec),
  305	format(atom(Id), '~q', [Spec]),
  306	cpack_uri(file_ref, Id, URI),
  307	(   rdf(URI, rdf:type, cpack:'FileRef')
  308	->  true
  309	;   cpack_uri(graph, 'file-references', Graph),
  310	    file_base(Spec, BaseName),
  311	    rdf_assert(URI, rdf:type, cpack:'FileRef', Graph),
  312	    rdf_assert(URI, cpack:name, literal(Id), Graph),
  313	    rdf_assert(URI, cpack:base, literal(BaseName), Graph),
  314	    resolve_file_ref(URI)
  315	).
  316
  317file_base(Spec, Base) :-
  318	atom(Spec), !,
  319	file_base_name(Spec, File),
  320	file_name_extension(Base, _Ext, File). % demand Prolog?
  321file_base(_/Rest, Base) :- !,
  322	file_base(Rest, Base).
  323file_base(Spec, Base) :-
  324	arg(1, Spec, Name),
  325	file_base(Name, Base).
 resolve_file(+File) is det
Create links to the FileRef objects to which this file may resolve.
  332resolve_file(File) :-
  333	rdf_has(File, cpack:base, Base),
  334	findall(FileRef,
  335		(   rdf_has(FileRef, cpack:base, Base),
  336		    rdfs_individual_of(FileRef, cpack:'FileRef')
  337		),
  338		Candidates),
  339	maplist(resolve_file_ref, Candidates).
 resolve_file_ref(+FileRef) is det
File files to which FileRef can resolve and add relate the file to this FileRef.
  347resolve_file_ref(FileRef) :-
  348	rdf_has(FileRef, cpack:name, literal(Id)),
  349	atom_to_term(Id, Spec, _Vars),
  350	forall(search_file(Spec, File),
  351	       assert_resolves(File, FileRef)).
  352
  353assert_resolves(File, FileRef) :-
  354	rdf_has(File, cpack:resolves, FileRef), !.
  355assert_resolves(File, FileRef) :-
  356	rdf_has(File, cpack:inPack, Pack),
  357	rdf_assert(File, cpack:resolves, FileRef, Pack).
  358
  359
  360		 /*******************************
  361		 *   HOOKS TO WORK ON GIT REPO	*
  362		 *******************************/
  363
  364:- multifile
  365	prolog:xref_open_source/2,
  366	prolog:xref_source_identifier/2,
  367	prolog:xref_source_file/3.  368
  369prolog:xref_open_source(File, Stream) :-
  370	rdf_is_resource(File),
  371	rdf_has(File, cpack:path, literal(Path)),
  372	rdf_has(File, cpack:inPack, Pack),
  373	(   rdf_has(File, cpack:mirrorRepository, Mirror),
  374	    rdf_has(Mirror, cpack:branch, literal(Branch))
  375	->  true
  376	;   Branch = master
  377	),
  378	cpack_our_mirror(Pack, BareGitDir),
  379	git_open_file(BareGitDir, Path, Branch, Stream).
  380
  381prolog:xref_source_identifier(File, File) :-
  382	rdf_is_resource(File),
  383	rdfs_individual_of(File, cpack:'PrologFile').
 prolog:xref_source_file(+Spec, -File, +Options) is semidet
True when File is the URI of a file referenced by Spec. This predicate hooks into xref_source_file/3, making it possible for the cross-referencer to analyse files in GIT repositories.
  391prolog:xref_source_file(Spec, File, _Options) :-
  392	xref_git,
  393	rdf_is_resource(Spec),
  394	(   File = Spec
  395	;   prolog_file_type(Ext, prolog),
  396	    file_name_extension(Spec, Ext, File)
  397	),
  398	rdfs_individual_of(File, cpack:'File'), !.
  399prolog:xref_source_file(Spec, File, Options) :-
  400	xref_git,
  401	atom(Spec),
  402	option(relative_to(Base), Options),
  403	uri_normalized(Spec, Base, Spec2),
  404	(   File = Spec2
  405	;   prolog_file_type(Ext, prolog),
  406	    file_name_extension(Spec2, Ext, File)
  407	),
  408	rdfs_individual_of(File, cpack:'File'), !.
  409prolog:xref_source_file(Spec, File, _Options) :-
  410	xref_git,
  411	search_file(Spec, File), !,
  412					% but NOT a ClioPatria file
  413	\+ ( path_rule(Spec, cpacks(Segments)),
  414	     path_segments_atom(Segments, InPack),
  415	     absolute_file_name(cliopatria(InPack), _,
  416				[ file_type(prolog),
  417				  access(read),
  418				  file_errors(fail)
  419				])).
  420
  421
  422		 /*******************************
  423		 *	      MESSAGES		*
  424		 *******************************/
  425
  426:- multifile
  427	prolog:message//1.  428
  429prolog:message(cpack(xref(File))) -->
  430	{ rdf_has(File, cpack:path, literal(Path)) },
  431	[ 'Analyzing ~w'-[Path] ]