"Fossies" - the Fresh Open Source Software Archive  

Source code changes of the file "service/staticAnalysis.ml" between
pyre-check-0.0.53.tar.gz and pyre-check-0.0.54.tar.gz

About: Pyre is a performant type checker for Python (ships with Pysa, a security focused static analysis tool).

staticAnalysis.ml  (pyre-check-0.0.53):staticAnalysis.ml  (pyre-check-0.0.54)
skipping to change at line 105 skipping to change at line 105
in in
let is_internal_source = let is_internal_source =
Ast.SourcePath.is_internal_path Ast.SourcePath.is_internal_path
~configuration ~configuration
(Ast.SourcePath.full_path ~configuration source.source_path) (Ast.SourcePath.full_path ~configuration source.source_path)
in in
( List.map included ~f:(fun (callable, define) -> ( List.map included ~f:(fun (callable, define) ->
{ callable; define; is_internal = is_internal_source }), { callable; define; is_internal = is_internal_source }),
filtered ) filtered )
let get_source ~environment qualifier =
let ast_environment = TypeEnvironment.ReadOnly.ast_environment environment in
AstEnvironment.ReadOnly.get_processed_source ast_environment qualifier
let fetch_callables_to_analyze ~scheduler ~environment ~configuration ~qualifier
s =
let global_resolution = TypeEnvironment.ReadOnly.global_resolution environment
in
let classify_source
(callables, stubs)
{ callable; define = { Node.value = define; _ }; is_internal }
=
if Define.is_stub define then
callables, callable :: stubs
else
(callable, is_internal) :: callables, stubs
in
let map result qualifiers =
let make_callables
((existing_callables, existing_stubs, filtered_callables) as result)
qualifier
=
get_source ~environment qualifier
>>| (fun source ->
let callables, new_filtered_callables =
regular_and_filtered_callables ~configuration ~resolution:global_r
esolution ~source
in
let callables, stubs =
List.fold callables ~f:classify_source ~init:(existing_callables,
existing_stubs)
in
let updated_filtered_callables =
List.fold
new_filtered_callables
~init:filtered_callables
~f:(Fn.flip Callable.Set.add)
in
callables, stubs, updated_filtered_callables)
|> Option.value ~default:result
in
List.fold qualifiers ~f:make_callables ~init:result
in
let reduce
(new_callables, new_stubs, new_filtered_callables)
(callables, stubs, filtered_callables)
=
( List.rev_append new_callables callables,
List.rev_append new_stubs stubs,
Callable.Set.union new_filtered_callables filtered_callables )
in
Scheduler.map_reduce
scheduler
~policy:
(Scheduler.Policy.fixed_chunk_count ~minimum_chunk_size:50 ~preferred_chun
ks_per_worker:1 ())
~map
~reduce
~initial:([], [], Callable.Set.empty)
~inputs:qualifiers
()
let record_overrides_for_qualifiers ~scheduler ~environment ~skip_overrides ~qua
lifiers =
let overrides =
let combine ~key:_ left right = List.rev_append left right in
let build_overrides overrides qualifier =
try
match get_source ~environment qualifier with
| None -> overrides
| Some source ->
let new_overrides =
DependencyGraph.create_overrides ~environment ~source
|> Reference.Map.filter_keys ~f:(fun override ->
not (Reference.Set.mem skip_overrides override))
in
Map.merge_skewed overrides new_overrides ~combine
with
| ClassHierarchy.Untracked untracked_type ->
Log.warning
"Error building overrides in path %a for untracked type %a"
Reference.pp
qualifier
Type.pp
untracked_type;
overrides
in
Scheduler.map_reduce
scheduler
~policy:(Scheduler.Policy.legacy_fixed_chunk_count ())
~initial:DependencyGraph.empty_overrides
~map:(fun _ qualifiers ->
List.fold qualifiers ~init:DependencyGraph.empty_overrides ~f:build_over
rides)
~reduce:(Map.merge_skewed ~combine)
~inputs:qualifiers
()
in
let {
Taint.TaintConfiguration.analysis_model_constraints = { maximum_overrides_to
_analyze; _ };
_;
}
=
Taint.TaintConfiguration.get ()
in
record_overrides ?maximum_overrides_to_analyze overrides;
overrides
let analyze let analyze
~scheduler ~scheduler
~analysis_kind ~analysis_kind
~configuration: ~configuration:
( { ( {
Configuration.StaticAnalysis.configuration; Configuration.StaticAnalysis.configuration;
dump_call_graph; dump_call_graph;
verify_models; verify_models;
rule_filter; rule_filter;
find_obscure_flows; find_obscure_flows;
_; _;
} as analysis_configuration ) } as analysis_configuration )
~filename_lookup ~filename_lookup
~environment ~environment
~qualifiers ~qualifiers
() ()
= =
let global_resolution = TypeEnvironment.ReadOnly.global_resolution environment let get_source = get_source ~environment in
in
let get_source qualifier =
let ast_environment = TypeEnvironment.ReadOnly.ast_environment environment i
n
AstEnvironment.ReadOnly.get_processed_source ast_environment qualifier
in
let timer = Timer.start () in let timer = Timer.start () in
Log.info "Fetching initial callables to analyze..."; Log.info "Fetching initial callables to analyze...";
let callables_with_dependency_information, stubs, filtered_callables = let callables_with_dependency_information, stubs, filtered_callables =
let classify_source fetch_callables_to_analyze ~scheduler ~environment ~configuration ~qualifier
(callables, stubs) s
{ callable; define = { Node.value = define; _ }; is_internal }
=
if Define.is_stub define then
callables, callable :: stubs
else
(callable, is_internal) :: callables, stubs
in
let map result qualifiers =
let make_callables
((existing_callables, existing_stubs, filtered_callables) as result)
qualifier
=
get_source qualifier
>>| (fun source ->
let callables, new_filtered_callables =
regular_and_filtered_callables ~configuration ~resolution:global
_resolution ~source
in
let callables, stubs =
List.fold callables ~f:classify_source ~init:(existing_callables
, existing_stubs)
in
let updated_filtered_callables =
List.fold
new_filtered_callables
~init:filtered_callables
~f:(Fn.flip Callable.Set.add)
in
callables, stubs, updated_filtered_callables)
|> Option.value ~default:result
in
List.fold qualifiers ~f:make_callables ~init:result
in
let reduce
(new_callables, new_stubs, new_filtered_callables)
(callables, stubs, filtered_callables)
=
( List.rev_append new_callables callables,
List.rev_append new_stubs stubs,
Callable.Set.union new_filtered_callables filtered_callables )
in
Scheduler.map_reduce
scheduler
~policy:
(Scheduler.Policy.fixed_chunk_count
~minimum_chunk_size:50
~preferred_chunks_per_worker:1
())
~map
~reduce
~initial:([], [], Callable.Set.empty)
~inputs:qualifiers
()
in in
let stubs = (stubs :> Callable.t list) in let stubs = (stubs :> Callable.t list) in
Statistics.performance ~name:"Fetched initial callables to analyze" ~timer (); Statistics.performance ~name:"Fetched initial callables to analyze" ~timer ();
let analyses = [analysis_kind] in let analyses = [analysis_kind] in
let timer = Timer.start () in let timer = Timer.start () in
Log.info "Initializing analysis..."; Log.info "Initializing analysis...";
(* Initialize and add initial models of analyses to shared mem. *) (* Initialize and add initial models of analyses to shared mem. *)
let skip_overrides = let skip_overrides =
let configuration_json = let configuration_json =
let taint_model_paths = let taint_model_paths =
skipping to change at line 233 skipping to change at line 278
~functions ~functions
~stubs ~stubs
in in
Analysis.record_initial_models ~functions ~stubs models; Analysis.record_initial_models ~functions ~stubs models;
skip_overrides skip_overrides
in in
Statistics.performance ~name:"Computed initial analysis state" ~timer (); Statistics.performance ~name:"Computed initial analysis state" ~timer ();
Log.info "Recording overrides..."; Log.info "Recording overrides...";
let timer = Timer.start () in let timer = Timer.start () in
let overrides = let overrides =
let combine ~key:_ left right = List.rev_append left right in record_overrides_for_qualifiers ~scheduler ~environment ~skip_overrides ~qua
let build_overrides overrides qualifier = lifiers
try
match get_source qualifier with
| None -> overrides
| Some source ->
let new_overrides =
DependencyGraph.create_overrides ~environment ~source
|> Reference.Map.filter_keys ~f:(fun override ->
not (Reference.Set.mem skip_overrides override))
in
Map.merge_skewed overrides new_overrides ~combine
with
| ClassHierarchy.Untracked untracked_type ->
Log.warning
"Error building overrides in path %a for untracked type %a"
Reference.pp
qualifier
Type.pp
untracked_type;
overrides
in
Scheduler.map_reduce
scheduler
~policy:(Scheduler.Policy.legacy_fixed_chunk_count ())
~initial:DependencyGraph.empty_overrides
~map:(fun _ qualifiers ->
List.fold qualifiers ~init:DependencyGraph.empty_overrides ~f:build_over
rides)
~reduce:(Map.merge_skewed ~combine)
~inputs:qualifiers
()
in in
let {
Taint.TaintConfiguration.analysis_model_constraints = { maximum_overrides_to
_analyze; _ };
_;
}
=
Taint.TaintConfiguration.get ()
in
record_overrides ?maximum_overrides_to_analyze overrides;
let override_dependencies = DependencyGraph.from_overrides overrides in let override_dependencies = DependencyGraph.from_overrides overrides in
Statistics.performance ~name:"Overrides recorded" ~timer (); Statistics.performance ~name:"Overrides recorded" ~timer ();
(* It's imperative that the call graph is built after the overrides are, due t o a hidden global (* It's imperative that the call graph is built after the overrides are, due t o a hidden global
state dependency. We rely on shared memory to tell us which methods are ove rridden to state dependency. We rely on shared memory to tell us which methods are ove rridden to
accurately model the call graph's overrides. Without it, we'll underanalyze and have an accurately model the call graph's overrides. Without it, we'll underanalyze and have an
inconsistent fixpoint. *) inconsistent fixpoint. *)
Log.info "Building call graph..."; Log.info "Building call graph...";
let timer = Timer.start () in let timer = Timer.start () in
let callgraph = let callgraph =
skipping to change at line 341 skipping to change at line 348
in in
List.iter override_targets ~f:add_predefined List.iter override_targets ~f:add_predefined
in in
Statistics.performance ~name:"Computed overrides" ~timer (); Statistics.performance ~name:"Computed overrides" ~timer ();
let all_callables = List.rev_append override_targets callables in let all_callables = List.rev_append override_targets callables in
Log.info Log.info
"Analysis fixpoint started for %d overrides %d functions..." "Analysis fixpoint started for %d overrides %d functions..."
(List.length override_targets) (List.length override_targets)
(List.length callables); (List.length callables);
let timer = Timer.start () in let timer = Timer.start () in
let save_results () =
Interprocedural.Analysis.save_results
~configuration:analysis_configuration
~filename_lookup
~analyses
all_callables
in
let () = let () =
try try
let iterations = let iterations =
Interprocedural.Analysis.compute_fixpoint Interprocedural.Analysis.compute_fixpoint
~scheduler ~scheduler
~environment ~environment
~analyses ~analyses
~dependencies ~dependencies
~filtered_callables ~filtered_callables
~all_callables ~all_callables
Interprocedural.Fixpoint.Epoch.initial Interprocedural.Fixpoint.Epoch.initial
in in
Log.info "Fixpoint iterations: %d" iterations Log.info "Fixpoint iterations: %d" iterations
with with
| exn -> | exn ->
Interprocedural.Analysis.save_results save_results ();
~configuration:analysis_configuration
~filename_lookup
~analyses
all_callables;
raise exn raise exn
in in
let () = save_results ();
Interprocedural.Analysis.save_results
~configuration:analysis_configuration
~filename_lookup
~analyses
all_callables
in
let errors = Interprocedural.Analysis.extract_errors scheduler all_callables i n let errors = Interprocedural.Analysis.extract_errors scheduler all_callables i n
Statistics.performance ~name:"Analysis fixpoint complete" ~timer (); Statistics.performance ~name:"Analysis fixpoint complete" ~timer ();
(* If saving to a file, don't return errors. Thousands of errors on output is inconvenient *) (* If saving to a file, don't return errors. Thousands of errors on output is inconvenient *)
if Option.is_some analysis_configuration.result_json_path then if Option.is_some analysis_configuration.result_json_path then
[] []
else else
errors errors
 End of changes. 8 change blocks. 
115 lines changed or deleted 123 lines changed or added

Home  |  About  |  Features  |  All  |  Newest  |  Dox  |  Diffs  |  RSS Feeds  |  Screenshots  |  Comments  |  Imprint  |  Privacy  |  HTTP(S)