"Fossies" - the Fresh Open Source Software Archive 
Member "PowerShell-7.2.6/src/System.Management.Automation/engine/Modules/AnalysisCache.cs" (11 Aug 2022, 50256 Bytes) of package /linux/misc/PowerShell-7.2.6.tar.gz:
As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C# source code syntax highlighting (style:
standard) with prefixed line numbers and
code folding option.
Alternatively you can here
view or
download the uninterpreted source code file.
For more information about "AnalysisCache.cs" see the
Fossies "Dox" file reference documentation.
1 // Copyright (c) Microsoft Corporation.
2 // Licensed under the MIT License.
3
4 using System.Collections;
5 using System.Collections.Concurrent;
6 using System.Collections.Generic;
7 using System.Diagnostics;
8 using System.Globalization;
9 using System.IO;
10 using System.Management.Automation.Internal;
11 using System.Management.Automation.Language;
12 using System.Management.Automation.Runspaces;
13 using System.Text;
14 using System.Threading;
15 using System.Threading.Tasks;
16
17 using Microsoft.PowerShell.Commands;
18
19 namespace System.Management.Automation
20 {
21 /// <summary>
22 /// Class to manage the caching of analysis data.
23 /// For performance, module command caching is flattened after discovery. Many modules have nested
24 /// modules that can only be resolved at runtime - for example,
25 /// script modules that declare: $env:PATH += "; $psScriptRoot". When
26 /// doing initial analysis, we include these in 'ExportedCommands'.
27 /// Changes to these type of modules will not be re-analyzed, unless the user re-imports the module,
28 /// or runs Get-Module -List.
29 /// </summary>
30 internal static class AnalysisCache
31 {
32 private static readonly AnalysisCacheData s_cacheData = AnalysisCacheData.Get();
33
34 // This dictionary shouldn't see much use, so low concurrency and capacity
35 private static readonly ConcurrentDictionary<string, string> s_modulesBeingAnalyzed =
36 new(concurrencyLevel: 1, capacity: 2, StringComparer.OrdinalIgnoreCase);
37
38 internal static readonly char[] InvalidCommandNameCharacters = new[]
39 {
40 '#', ',', '(', ')', '{', '}', '[', ']', '&', '/', '\\', '$', '^', ';', ':',
41 '"', '\'', '<', '>', '|', '?', '@', '`', '*', '%', '+', '=', '~'
42 };
43
44 internal static ConcurrentDictionary<string, CommandTypes> GetExportedCommands(string modulePath, bool testOnly, ExecutionContext context)
45 {
46 bool etwEnabled = CommandDiscoveryEventSource.Log.IsEnabled();
47 if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStart(modulePath);
48
49 DateTime lastWriteTime;
50 ModuleCacheEntry moduleCacheEntry;
51 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
52 {
53 if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);
54 return moduleCacheEntry.Commands;
55 }
56
57 ConcurrentDictionary<string, CommandTypes> result = null;
58
59 if (!testOnly)
60 {
61 var extension = Path.GetExtension(modulePath);
62 if (extension.Equals(StringLiterals.PowerShellDataFileExtension, StringComparison.OrdinalIgnoreCase))
63 {
64 result = AnalyzeManifestModule(modulePath, context, lastWriteTime, etwEnabled);
65 }
66 else if (extension.Equals(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))
67 {
68 result = AnalyzeScriptModule(modulePath, context, lastWriteTime);
69 }
70 else if (extension.Equals(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))
71 {
72 result = AnalyzeCdxmlModule(modulePath, context, lastWriteTime);
73 }
74 else if (extension.Equals(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))
75 {
76 result = AnalyzeDllModule(modulePath, context, lastWriteTime);
77 }
78 else if (extension.Equals(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))
79 {
80 result = AnalyzeDllModule(modulePath, context, lastWriteTime);
81 }
82 }
83
84 if (result != null)
85 {
86 s_cacheData.QueueSerialization();
87 ModuleIntrinsics.Tracer.WriteLine("Returning {0} exported commands.", result.Count);
88 }
89 else
90 {
91 ModuleIntrinsics.Tracer.WriteLine("Returning NULL for exported commands.");
92 }
93
94 if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);
95 return result;
96 }
97
98 private static ConcurrentDictionary<string, CommandTypes> AnalyzeManifestModule(string modulePath, ExecutionContext context, DateTime lastWriteTime, bool etwEnabled)
99 {
100 ConcurrentDictionary<string, CommandTypes> result = null;
101 try
102 {
103 var moduleManifestProperties = PsUtils.GetModuleManifestProperties(modulePath, PsUtils.FastModuleManifestAnalysisPropertyNames);
104 if (moduleManifestProperties != null)
105 {
106 if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && ModuleIsEditionIncompatible(modulePath, moduleManifestProperties))
107 {
108 ModuleIntrinsics.Tracer.WriteLine($"Module lies on the Windows System32 legacy module path and is incompatible with current PowerShell edition, skipping module: {modulePath}");
109 return null;
110 }
111
112 Version version;
113 if (ModuleUtils.IsModuleInVersionSubdirectory(modulePath, out version))
114 {
115 var versionInManifest = LanguagePrimitives.ConvertTo<Version>(moduleManifestProperties["ModuleVersion"]);
116 if (version != versionInManifest)
117 {
118 ModuleIntrinsics.Tracer.WriteLine("ModuleVersion in manifest does not match versioned module directory, skipping module: {0}", modulePath);
119 return null;
120 }
121 }
122
123 result = new ConcurrentDictionary<string, CommandTypes>(3, moduleManifestProperties.Count, StringComparer.OrdinalIgnoreCase);
124
125 var sawWildcard = false;
126 var hadCmdlets = AddPsd1EntryToResult(result, moduleManifestProperties["CmdletsToExport"], CommandTypes.Cmdlet, ref sawWildcard);
127 var hadFunctions = AddPsd1EntryToResult(result, moduleManifestProperties["FunctionsToExport"], CommandTypes.Function, ref sawWildcard);
128 var hadAliases = AddPsd1EntryToResult(result, moduleManifestProperties["AliasesToExport"], CommandTypes.Alias, ref sawWildcard);
129
130 var analysisSucceeded = hadCmdlets && hadFunctions && hadAliases;
131
132 if (!analysisSucceeded && !sawWildcard && (hadCmdlets || hadFunctions))
133 {
134 // If we're missing CmdletsToExport, that might still be OK, but only if we have a script module.
135 // Likewise, if we're missing FunctionsToExport, that might be OK, but only if we have a binary module.
136
137 analysisSucceeded = !CheckModulesTypesInManifestAgainstExportedCommands(moduleManifestProperties, hadCmdlets, hadFunctions, hadAliases);
138 }
139
140 if (analysisSucceeded)
141 {
142 var moduleCacheEntry = new ModuleCacheEntry
143 {
144 ModulePath = modulePath,
145 LastWriteTime = lastWriteTime,
146 Commands = result,
147 TypesAnalyzed = false,
148 Types = new ConcurrentDictionary<string, TypeAttributes>(1, 8, StringComparer.OrdinalIgnoreCase)
149 };
150 s_cacheData.Entries[modulePath] = moduleCacheEntry;
151 }
152 else
153 {
154 result = null;
155 }
156 }
157 }
158 catch (Exception e)
159 {
160 if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisException(modulePath, e.Message);
161 // Ignore the errors, proceed with the usual module analysis
162 ModuleIntrinsics.Tracer.WriteLine("Exception on fast-path analysis of module {0}", modulePath);
163 }
164
165 if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisResult(modulePath, result != null);
166
167 return result ?? AnalyzeTheOldWay(modulePath, context, lastWriteTime);
168 }
169
170 /// <summary>
171 /// Check if a module is compatible with the current PSEdition given its path and its manifest properties.
172 /// </summary>
173 /// <param name="modulePath">The path to the module.</param>
174 /// <param name="moduleManifestProperties">The properties of the module's manifest.</param>
175 /// <returns></returns>
176 internal static bool ModuleIsEditionIncompatible(string modulePath, Hashtable moduleManifestProperties)
177 {
178 #if UNIX
179 return false;
180 #else
181 if (!ModuleUtils.IsOnSystem32ModulePath(modulePath))
182 {
183 return false;
184 }
185
186 if (!moduleManifestProperties.ContainsKey("CompatiblePSEditions"))
187 {
188 return true;
189 }
190
191 return !Utils.IsPSEditionSupported(LanguagePrimitives.ConvertTo<string[]>(moduleManifestProperties["CompatiblePSEditions"]));
192 #endif
193 }
194
195 internal static bool ModuleAnalysisViaGetModuleRequired(object modulePathObj, bool hadCmdlets, bool hadFunctions, bool hadAliases)
196 {
197 if (!(modulePathObj is string modulePath))
198 return true;
199
200 if (modulePath.EndsWith(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))
201 {
202 // A script module can't exactly define cmdlets, but it can import a binary module (as nested), so
203 // it can indirectly define cmdlets. And obviously a script module can define functions and aliases.
204 // If we got here, one of those is missing, so analysis is required.
205 return true;
206 }
207
208 if (modulePath.EndsWith(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))
209 {
210 // A cdxml module can only define functions and aliases, so if we have both, no more analysis is required.
211 return !hadFunctions || !hadAliases;
212 }
213
214 if (modulePath.EndsWith(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))
215 {
216 // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,
217 // more analysis is required. If the module exports aliases, we can't discover that analyzing
218 // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.
219 return !hadCmdlets;
220 }
221
222 if (modulePath.EndsWith(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))
223 {
224 // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,
225 // more analysis is required. If the module exports aliases, we can't discover that analyzing
226 // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.
227 return !hadCmdlets;
228 }
229
230 // Any other extension (or no extension), just assume the worst and analyze the module
231 return true;
232 }
233
234 // Returns true if we need to analyze the manifest module in Get-Module because
235 // our quick and dirty module manifest analysis is missing something not easily
236 // discovered.
237 //
238 // TODO - psm1 modules are actually easily handled, so if we only saw a psm1 here,
239 // we should just analyze it and not fall back on Get-Module -List.
240 private static bool CheckModulesTypesInManifestAgainstExportedCommands(Hashtable moduleManifestProperties, bool hadCmdlets, bool hadFunctions, bool hadAliases)
241 {
242 var rootModule = moduleManifestProperties["RootModule"];
243 if (rootModule != null && ModuleAnalysisViaGetModuleRequired(rootModule, hadCmdlets, hadFunctions, hadAliases))
244 return true;
245
246 var moduleToProcess = moduleManifestProperties["ModuleToProcess"];
247 if (moduleToProcess != null && ModuleAnalysisViaGetModuleRequired(moduleToProcess, hadCmdlets, hadFunctions, hadAliases))
248 return true;
249
250 var nestedModules = moduleManifestProperties["NestedModules"];
251 if (nestedModules != null)
252 {
253 var nestedModule = nestedModules as string;
254 if (nestedModule != null)
255 {
256 return ModuleAnalysisViaGetModuleRequired(nestedModule, hadCmdlets, hadFunctions, hadAliases);
257 }
258
259 if (!(nestedModules is object[] nestedModuleArray))
260 return true;
261
262 foreach (var element in nestedModuleArray)
263 {
264 if (ModuleAnalysisViaGetModuleRequired(element, hadCmdlets, hadFunctions, hadAliases))
265 return true;
266 }
267 }
268
269 return false;
270 }
271
272 private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, string command, CommandTypes commandTypeToAdd, ref bool sawWildcard)
273 {
274 if (WildcardPattern.ContainsWildcardCharacters(command))
275 {
276 sawWildcard = true;
277 return false;
278 }
279
280 // An empty string is one way of saying "no exported commands".
281 if (command.Length != 0)
282 {
283 CommandTypes commandTypes;
284 if (result.TryGetValue(command, out commandTypes))
285 {
286 commandTypes |= commandTypeToAdd;
287 }
288 else
289 {
290 commandTypes = commandTypeToAdd;
291 }
292
293 result[command] = commandTypes;
294 }
295
296 return true;
297 }
298
299 private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, object value, CommandTypes commandTypeToAdd, ref bool sawWildcard)
300 {
301 string command = value as string;
302 if (command != null)
303 {
304 return AddPsd1EntryToResult(result, command, commandTypeToAdd, ref sawWildcard);
305 }
306
307 object[] commands = value as object[];
308 if (commands != null)
309 {
310 foreach (var o in commands)
311 {
312 if (!AddPsd1EntryToResult(result, o, commandTypeToAdd, ref sawWildcard))
313 return false;
314 }
315
316 // An empty array is still success, that's how a manifest declares that
317 // no entries are exported (unlike the lack of an entry, or $null).
318 return true;
319 }
320
321 // Unknown type, let Get-Module -List deal with this manifest
322 return false;
323 }
324
325 private static ConcurrentDictionary<string, CommandTypes> AnalyzeScriptModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
326 {
327 var scriptAnalysis = ScriptAnalysis.Analyze(modulePath, context);
328 if (scriptAnalysis == null)
329 {
330 return null;
331 }
332
333 List<WildcardPattern> scriptAnalysisPatterns = new List<WildcardPattern>();
334 foreach (string discoveredCommandFilter in scriptAnalysis.DiscoveredCommandFilters)
335 {
336 scriptAnalysisPatterns.Add(new WildcardPattern(discoveredCommandFilter));
337 }
338
339 var result = new ConcurrentDictionary<string, CommandTypes>(3,
340 scriptAnalysis.DiscoveredExports.Count + scriptAnalysis.DiscoveredAliases.Count,
341 StringComparer.OrdinalIgnoreCase);
342
343 // Add any directly discovered exports
344 foreach (var command in scriptAnalysis.DiscoveredExports)
345 {
346 if (SessionStateUtilities.MatchesAnyWildcardPattern(command, scriptAnalysisPatterns, true))
347 {
348 if (command.IndexOfAny(InvalidCommandNameCharacters) < 0)
349 {
350 result[command] = CommandTypes.Function;
351 }
352 }
353 }
354
355 // Add the discovered aliases
356 foreach (var pair in scriptAnalysis.DiscoveredAliases)
357 {
358 var commandName = pair.Key;
359 // These are already filtered
360 if (commandName.IndexOfAny(InvalidCommandNameCharacters) < 0)
361 {
362 result.AddOrUpdate(commandName, CommandTypes.Alias,
363 static (_, existingCommandType) => existingCommandType | CommandTypes.Alias);
364 }
365 }
366
367 // Add any files in PsScriptRoot if it added itself to the path
368 if (scriptAnalysis.AddsSelfToPath)
369 {
370 string baseDirectory = Path.GetDirectoryName(modulePath);
371
372 try
373 {
374 foreach (string item in Directory.EnumerateFiles(baseDirectory, "*.ps1"))
375 {
376 var command = Path.GetFileNameWithoutExtension(item);
377 result.AddOrUpdate(command, CommandTypes.ExternalScript,
378 static (_, existingCommandType) => existingCommandType | CommandTypes.ExternalScript);
379 }
380 }
381 catch (UnauthorizedAccessException)
382 {
383 // Consume this exception here
384 }
385 }
386
387 ConcurrentDictionary<string, TypeAttributes> exportedClasses = new(
388 concurrencyLevel: 1,
389 capacity: scriptAnalysis.DiscoveredClasses.Count,
390 StringComparer.OrdinalIgnoreCase);
391 foreach (var exportedClass in scriptAnalysis.DiscoveredClasses)
392 {
393 exportedClasses[exportedClass.Name] = exportedClass.TypeAttributes;
394 }
395
396 var moduleCacheEntry = new ModuleCacheEntry
397 {
398 ModulePath = modulePath,
399 LastWriteTime = lastWriteTime,
400 Commands = result,
401 TypesAnalyzed = true,
402 Types = exportedClasses
403 };
404 s_cacheData.Entries[modulePath] = moduleCacheEntry;
405
406 return result;
407 }
408
409 private static ConcurrentDictionary<string, CommandTypes> AnalyzeCdxmlModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
410 {
411 return AnalyzeTheOldWay(modulePath, context, lastWriteTime);
412 }
413
414 private static ConcurrentDictionary<string, CommandTypes> AnalyzeDllModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
415 {
416 return AnalyzeTheOldWay(modulePath, context, lastWriteTime);
417 }
418
419 private static ConcurrentDictionary<string, CommandTypes> AnalyzeTheOldWay(string modulePath, ExecutionContext context, DateTime lastWriteTime)
420 {
421 try
422 {
423 // If we're already analyzing this module, let the recursion bottom out.
424 if (!s_modulesBeingAnalyzed.TryAdd(modulePath, modulePath))
425 {
426 ModuleIntrinsics.Tracer.WriteLine("{0} is already being analyzed. Exiting.", modulePath);
427 return null;
428 }
429
430 // Record that we're analyzing this specific module so that we don't get stuck in recursion
431 ModuleIntrinsics.Tracer.WriteLine("Started analysis: {0}", modulePath);
432 CallGetModuleDashList(context, modulePath);
433
434 ModuleCacheEntry moduleCacheEntry;
435 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
436 {
437 return moduleCacheEntry.Commands;
438 }
439 }
440 catch (Exception e)
441 {
442 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
443
444 // Catch-all OK, third-party call-out.
445 }
446 finally
447 {
448 ModuleIntrinsics.Tracer.WriteLine("Finished analysis: {0}", modulePath);
449 s_modulesBeingAnalyzed.TryRemove(modulePath, out modulePath);
450 }
451
452 return null;
453 }
454
455 /// <summary>
456 /// Return the exported types for a specific module.
457 /// If the module is already cache, return from cache, else cache the module.
458 /// Also re-cache the module if the cached item is stale.
459 /// </summary>
460 /// <param name="modulePath">Path to the module to get exported types from.</param>
461 /// <param name="context">Current Context.</param>
462 /// <returns></returns>
463 internal static ConcurrentDictionary<string, TypeAttributes> GetExportedClasses(string modulePath, ExecutionContext context)
464 {
465 DateTime lastWriteTime;
466 ModuleCacheEntry moduleCacheEntry;
467 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry) && moduleCacheEntry.TypesAnalyzed)
468 {
469 return moduleCacheEntry.Types;
470 }
471
472 try
473 {
474 CallGetModuleDashList(context, modulePath);
475 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
476 {
477 return moduleCacheEntry.Types;
478 }
479 }
480 catch (Exception e)
481 {
482 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
483
484 // Catch-all OK, third-party call-out.
485 }
486
487 return null;
488 }
489
490 internal static void CacheModuleExports(PSModuleInfo module, ExecutionContext context)
491 {
492 ModuleIntrinsics.Tracer.WriteLine("Requested caching for {0}", module.Name);
493
494 // Don't cache incompatible modules on the system32 module path even if loaded with
495 // -SkipEditionCheck, since it will break subsequent sessions
496 if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && !module.IsConsideredEditionCompatible)
497 {
498 ModuleIntrinsics.Tracer.WriteLine($"Module '{module.Name}' not edition compatible and not cached.");
499 return;
500 }
501
502 DateTime lastWriteTime;
503 ModuleCacheEntry moduleCacheEntry;
504 GetModuleEntryFromCache(module.Path, out lastWriteTime, out moduleCacheEntry);
505
506 var realExportedCommands = module.ExportedCommands;
507 var realExportedClasses = module.GetExportedTypeDefinitions();
508 ConcurrentDictionary<string, CommandTypes> exportedCommands;
509 ConcurrentDictionary<string, TypeAttributes> exportedClasses;
510
511 // First see if the existing module info is sufficient. GetModuleEntryFromCache does LastWriteTime
512 // verification, so this will also return nothing if the cache is out of date or corrupt.
513 if (moduleCacheEntry != null)
514 {
515 bool needToUpdate = false;
516
517 // We need to iterate and check as exportedCommands will have more item as it can have aliases as well.
518 exportedCommands = moduleCacheEntry.Commands;
519 foreach (var pair in realExportedCommands)
520 {
521 var commandName = pair.Key;
522 var realCommandType = pair.Value.CommandType;
523 CommandTypes commandType;
524 if (!exportedCommands.TryGetValue(commandName, out commandType) || commandType != realCommandType)
525 {
526 needToUpdate = true;
527 break;
528 }
529 }
530
531 exportedClasses = moduleCacheEntry.Types;
532 foreach (var pair in realExportedClasses)
533 {
534 var className = pair.Key;
535 var realTypeAttributes = pair.Value.TypeAttributes;
536 TypeAttributes typeAttributes;
537 if (!exportedClasses.TryGetValue(className, out typeAttributes) ||
538 typeAttributes != realTypeAttributes)
539 {
540 needToUpdate = true;
541 break;
542 }
543 }
544
545 // Update or not, we've analyzed commands and types now.
546 moduleCacheEntry.TypesAnalyzed = true;
547
548 if (!needToUpdate)
549 {
550 ModuleIntrinsics.Tracer.WriteLine("Existing cached info up-to-date. Skipping.");
551 return;
552 }
553
554 exportedCommands.Clear();
555 exportedClasses.Clear();
556 }
557 else
558 {
559 exportedCommands = new ConcurrentDictionary<string, CommandTypes>(3, realExportedCommands.Count, StringComparer.OrdinalIgnoreCase);
560 exportedClasses = new ConcurrentDictionary<string, TypeAttributes>(1, realExportedClasses.Count, StringComparer.OrdinalIgnoreCase);
561 moduleCacheEntry = new ModuleCacheEntry
562 {
563 ModulePath = module.Path,
564 LastWriteTime = lastWriteTime,
565 Commands = exportedCommands,
566 TypesAnalyzed = true,
567 Types = exportedClasses
568 };
569 moduleCacheEntry = s_cacheData.Entries.GetOrAdd(module.Path, moduleCacheEntry);
570 }
571
572 // We need to update the cache
573 foreach (var exportedCommand in realExportedCommands.Values)
574 {
575 ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", exportedCommand.Name);
576 exportedCommands.GetOrAdd(exportedCommand.Name, exportedCommand.CommandType);
577 }
578
579 foreach (var pair in realExportedClasses)
580 {
581 var className = pair.Key;
582 ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", className);
583 moduleCacheEntry.Types.AddOrUpdate(className, pair.Value.TypeAttributes, (k, t) => t);
584 }
585
586 s_cacheData.QueueSerialization();
587 }
588
589 private static void CallGetModuleDashList(ExecutionContext context, string modulePath)
590 {
591 CommandInfo commandInfo = new CmdletInfo("Get-Module", typeof(GetModuleCommand), null, null, context);
592 Command getModuleCommand = new Command(commandInfo);
593
594 try
595 {
596 PowerShell.Create(RunspaceMode.CurrentRunspace)
597 .AddCommand(getModuleCommand)
598 .AddParameter("List", true)
599 .AddParameter("ErrorAction", ActionPreference.Ignore)
600 .AddParameter("WarningAction", ActionPreference.Ignore)
601 .AddParameter("InformationAction", ActionPreference.Ignore)
602 .AddParameter("Verbose", false)
603 .AddParameter("Debug", false)
604 .AddParameter("Name", modulePath)
605 .Invoke();
606 }
607 catch (Exception e)
608 {
609 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
610
611 // Catch-all OK, third-party call-out.
612 }
613 }
614
615 private static bool GetModuleEntryFromCache(string modulePath, out DateTime lastWriteTime, out ModuleCacheEntry moduleCacheEntry)
616 {
617 try
618 {
619 lastWriteTime = new FileInfo(modulePath).LastWriteTime;
620 }
621 catch (Exception e)
622 {
623 ModuleIntrinsics.Tracer.WriteLine("Exception checking LastWriteTime on module {0}: {1}", modulePath, e.Message);
624 lastWriteTime = DateTime.MinValue;
625 }
626
627 if (s_cacheData.Entries.TryGetValue(modulePath, out moduleCacheEntry))
628 {
629 if (lastWriteTime == moduleCacheEntry.LastWriteTime)
630 {
631 return true;
632 }
633
634 ModuleIntrinsics.Tracer.WriteLine("{0}: cache entry out of date, cached on {1}, last updated on {2}",
635 modulePath, moduleCacheEntry.LastWriteTime, lastWriteTime);
636
637 s_cacheData.Entries.TryRemove(modulePath, out moduleCacheEntry);
638 }
639
640 moduleCacheEntry = null;
641 return false;
642 }
643 }
644
645 internal sealed class AnalysisCacheData
646 {
647 private static byte[] GetHeader()
648 {
649 return new byte[]
650 {
651 0x50, 0x53, 0x4d, 0x4f, 0x44, 0x55, 0x4c, 0x45, 0x43, 0x41, 0x43, 0x48, 0x45, // PSMODULECACHE
652 0x01 // version #
653 };
654 }
655
656 // The last time the index was maintained.
657 public DateTime LastReadTime { get; set; }
658
659 public ConcurrentDictionary<string, ModuleCacheEntry> Entries { get; set; }
660
661 private int _saveCacheToDiskQueued;
662
663 private bool _saveCacheToDisk = true;
664
665 public void QueueSerialization()
666 {
667 // We expect many modules to rapidly call for serialization.
668 // Instead of doing it right away, we'll queue a task that starts writing
669 // after it seems like we've stopped adding stuff to write out. This is
670 // avoids blocking the pipeline thread waiting for the write to finish.
671 // We want to make sure we only queue one task.
672 if (_saveCacheToDisk && Interlocked.Increment(ref _saveCacheToDiskQueued) == 1)
673 {
674 Task.Run(async delegate
675 {
676 // Wait a while before assuming we've finished the updates,
677 // writing the cache out in a timely matter isn't too important
678 // now anyway.
679 await Task.Delay(10000).ConfigureAwait(false);
680 int counter1, counter2;
681 do
682 {
683 // Check the counter a couple times with a delay,
684 // if it's stable, then proceed with writing.
685 counter1 = _saveCacheToDiskQueued;
686 await Task.Delay(3000).ConfigureAwait(false);
687 counter2 = _saveCacheToDiskQueued;
688 } while (counter1 != counter2);
689 Serialize(s_cacheStoreLocation);
690 });
691 }
692 }
693
694 // Remove entries that are not needed anymore, e.g. if a module was removed.
695 // If anything is removed, save the cache.
696 private void Cleanup()
697 {
698 Diagnostics.Assert(Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null,
699 "Caller to check environment variable before calling");
700
701 bool removedSomething = false;
702 var keys = Entries.Keys;
703 foreach (var key in keys)
704 {
705 if (!File.Exists(key))
706 {
707 removedSomething |= Entries.TryRemove(key, out ModuleCacheEntry _);
708 }
709 }
710
711 if (removedSomething)
712 {
713 QueueSerialization();
714 }
715 }
716
717 private static unsafe void Write(int val, byte[] bytes, FileStream stream)
718 {
719 Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");
720 fixed (byte* b = bytes) *((int*)b) = val;
721 stream.Write(bytes, 0, 4);
722 }
723
724 private static unsafe void Write(long val, byte[] bytes, FileStream stream)
725 {
726 Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");
727 fixed (byte* b = bytes) *((long*)b) = val;
728 stream.Write(bytes, 0, 8);
729 }
730
731 private static void Write(string val, byte[] bytes, FileStream stream)
732 {
733 Write(val.Length, bytes, stream);
734 bytes = Encoding.UTF8.GetBytes(val);
735 stream.Write(bytes, 0, bytes.Length);
736 }
737
738 private void Serialize(string filename)
739 {
740 AnalysisCacheData fromOtherProcess = null;
741 Diagnostics.Assert(_saveCacheToDisk, "Serialize should never be called without going through QueueSerialization which has a check");
742
743 try
744 {
745 if (File.Exists(filename))
746 {
747 var fileLastWriteTime = new FileInfo(filename).LastWriteTime;
748 if (fileLastWriteTime > this.LastReadTime)
749 {
750 fromOtherProcess = Deserialize(filename);
751 }
752 }
753 else
754 {
755 // Make sure the folder exists
756 var folder = Path.GetDirectoryName(filename);
757 if (!Directory.Exists(folder))
758 {
759 try
760 {
761 Directory.CreateDirectory(folder);
762 }
763 catch (UnauthorizedAccessException)
764 {
765 // service accounts won't be able to create directory
766 _saveCacheToDisk = false;
767 return;
768 }
769 }
770 }
771 }
772 catch (Exception e)
773 {
774 ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache {0}: {1} ", filename, e.Message);
775 }
776
777 if (fromOtherProcess != null)
778 {
779 // We should merge with what another process wrote so we don't clobber useful analysis
780 foreach (var otherEntryPair in fromOtherProcess.Entries)
781 {
782 var otherModuleName = otherEntryPair.Key;
783 var otherEntry = otherEntryPair.Value;
784 ModuleCacheEntry thisEntry;
785 if (Entries.TryGetValue(otherModuleName, out thisEntry))
786 {
787 if (otherEntry.LastWriteTime > thisEntry.LastWriteTime)
788 {
789 // The other entry is newer, take it over ours
790 Entries[otherModuleName] = otherEntry;
791 }
792 }
793 else
794 {
795 Entries[otherModuleName] = otherEntry;
796 }
797 }
798 }
799
800 // "PSMODULECACHE" -> 13 bytes
801 // byte ( 1 byte) -> version
802 // int ( 4 bytes) -> count of entries
803 // entries (?? bytes) -> all entries
804 //
805 // each entry is
806 // DateTime ( 8 bytes) -> last write time for module file
807 // int ( 4 bytes) -> path length
808 // string (?? bytes) -> utf8 encoded path
809 // int ( 4 bytes) -> count of commands
810 // commands (?? bytes) -> all commands
811 // int ( 4 bytes) -> count of types, -1 means unanalyzed (and 0 items serialized)
812 // types (?? bytes) -> all types
813 //
814 // each command is
815 // int ( 4 bytes) -> command name length
816 // string (?? bytes) -> utf8 encoded command name
817 // int ( 4 bytes) -> CommandTypes enum
818 //
819 // each type is
820 // int ( 4 bytes) -> type name length
821 // string (?? bytes) -> utf8 encoded type name
822 // int ( 4 bytes) -> type attributes
823 try
824 {
825 var bytes = new byte[8];
826
827 using (var stream = File.Create(filename))
828 {
829 var headerBytes = GetHeader();
830 stream.Write(headerBytes, 0, headerBytes.Length);
831
832 // Count of entries
833 Write(Entries.Count, bytes, stream);
834
835 foreach (var pair in Entries.ToArray())
836 {
837 var path = pair.Key;
838 var entry = pair.Value;
839
840 // Module last write time
841 Write(entry.LastWriteTime.Ticks, bytes, stream);
842
843 // Module path
844 Write(path, bytes, stream);
845
846 // Commands
847 var commandPairs = entry.Commands.ToArray();
848 Write(commandPairs.Length, bytes, stream);
849
850 foreach (var command in commandPairs)
851 {
852 Write(command.Key, bytes, stream);
853 Write((int)command.Value, bytes, stream);
854 }
855
856 // Types
857 var typePairs = entry.Types.ToArray();
858 Write(entry.TypesAnalyzed ? typePairs.Length : -1, bytes, stream);
859
860 foreach (var type in typePairs)
861 {
862 Write(type.Key, bytes, stream);
863 Write((int)type.Value, bytes, stream);
864 }
865 }
866 }
867 // We just wrote the file, note this so we can detect writes from another process
868 LastReadTime = new FileInfo(filename).LastWriteTime;
869 }
870 catch (Exception e)
871 {
872 ModuleIntrinsics.Tracer.WriteLine("Exception writing module analysis cache {0}: {1} ", filename, e.Message);
873 }
874
875 // Reset our counter so we can write again if asked.
876 Interlocked.Exchange(ref _saveCacheToDiskQueued, 0);
877 }
878
879 private const string TruncatedErrorMessage = "module cache file appears truncated";
880 private const string InvalidSignatureErrorMessage = "module cache signature not valid";
881 private const string PossibleCorruptionErrorMessage = "possible corruption in module cache";
882
883 private static unsafe long ReadLong(FileStream stream, byte[] bytes)
884 {
885 Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");
886 if (stream.Read(bytes, 0, 8) != 8)
887 throw new Exception(TruncatedErrorMessage);
888 fixed (byte* b = bytes)
889 return *(long*)b;
890 }
891
892 private static unsafe int ReadInt(FileStream stream, byte[] bytes)
893 {
894 Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");
895 if (stream.Read(bytes, 0, 4) != 4)
896 throw new Exception(TruncatedErrorMessage);
897 fixed (byte* b = bytes)
898 return *(int*)b;
899 }
900
901 private static string ReadString(FileStream stream, ref byte[] bytes)
902 {
903 int length = ReadInt(stream, bytes);
904 if (length > 10 * 1024)
905 throw new Exception(PossibleCorruptionErrorMessage);
906 if (length > bytes.Length)
907 bytes = new byte[length];
908 if (stream.Read(bytes, 0, length) != length)
909 throw new Exception(TruncatedErrorMessage);
910 return Encoding.UTF8.GetString(bytes, 0, length);
911 }
912
913 private static void ReadHeader(FileStream stream, byte[] bytes)
914 {
915 var headerBytes = GetHeader();
916 var length = headerBytes.Length;
917 Diagnostics.Assert(bytes.Length >= length, "must pass a large enough byte array");
918 if (stream.Read(bytes, 0, length) != length)
919 throw new Exception(TruncatedErrorMessage);
920
921 for (int i = 0; i < length; i++)
922 {
923 if (bytes[i] != headerBytes[i])
924 {
925 throw new Exception(InvalidSignatureErrorMessage);
926 }
927 }
928 // No need to return - we don't use it other than to detect the correct file format
929 }
930
931 public static AnalysisCacheData Deserialize(string filename)
932 {
933 using (var stream = File.OpenRead(filename))
934 {
935 var result = new AnalysisCacheData { LastReadTime = DateTime.Now };
936
937 var bytes = new byte[1024];
938
939 // Header
940 // "PSMODULECACHE" -> 13 bytes
941 // byte ( 1 byte) -> version
942 ReadHeader(stream, bytes);
943
944 // int ( 4 bytes) -> count of entries
945 int entries = ReadInt(stream, bytes);
946 if (entries > 20 * 1024)
947 throw new Exception(PossibleCorruptionErrorMessage);
948
949 result.Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, entries, StringComparer.OrdinalIgnoreCase);
950
951 // entries (?? bytes) -> all entries
952 while (entries > 0)
953 {
954 // DateTime ( 8 bytes) -> last write time for module file
955 var lastWriteTime = new DateTime(ReadLong(stream, bytes));
956
957 // int ( 4 bytes) -> path length
958 // string (?? bytes) -> utf8 encoded path
959 var path = ReadString(stream, ref bytes);
960
961 // int ( 4 bytes) -> count of commands
962 var countItems = ReadInt(stream, bytes);
963 if (countItems > 20 * 1024)
964 throw new Exception(PossibleCorruptionErrorMessage);
965
966 var commands = new ConcurrentDictionary<string, CommandTypes>(/*concurrency*/3, countItems, StringComparer.OrdinalIgnoreCase);
967
968 // commands (?? bytes) -> all commands
969 while (countItems > 0)
970 {
971 // int ( 4 bytes) -> command name length
972 // string (?? bytes) -> utf8 encoded command name
973 var commandName = ReadString(stream, ref bytes);
974
975 // int ( 4 bytes) -> CommandTypes enum
976 var commandTypes = (CommandTypes)ReadInt(stream, bytes);
977
978 // Ignore empty entries (possible corruption in the cache or bug?)
979 if (!string.IsNullOrWhiteSpace(commandName))
980 commands[commandName] = commandTypes;
981
982 countItems -= 1;
983 }
984
985 // int ( 4 bytes) -> count of types
986 countItems = ReadInt(stream, bytes);
987
988 bool typesAnalyzed = countItems != -1;
989 if (!typesAnalyzed)
990 countItems = 0;
991 if (countItems > 20 * 1024)
992 throw new Exception(PossibleCorruptionErrorMessage);
993
994 var types = new ConcurrentDictionary<string, TypeAttributes>(1, countItems, StringComparer.OrdinalIgnoreCase);
995
996 // types (?? bytes) -> all types
997 while (countItems > 0)
998 {
999 // int ( 4 bytes) -> type name length
1000 // string (?? bytes) -> utf8 encoded type name
1001 var typeName = ReadString(stream, ref bytes);
1002
1003 // int ( 4 bytes) -> type attributes
1004 var typeAttributes = (TypeAttributes)ReadInt(stream, bytes);
1005
1006 // Ignore empty entries (possible corruption in the cache or bug?)
1007 if (!string.IsNullOrWhiteSpace(typeName))
1008 types[typeName] = typeAttributes;
1009
1010 countItems -= 1;
1011 }
1012
1013 var entry = new ModuleCacheEntry
1014 {
1015 ModulePath = path,
1016 LastWriteTime = lastWriteTime,
1017 Commands = commands,
1018 TypesAnalyzed = typesAnalyzed,
1019 Types = types
1020 };
1021 result.Entries[path] = entry;
1022
1023 entries -= 1;
1024 }
1025
1026 if (Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null)
1027 {
1028 Task.Delay(10000).ContinueWith(_ => result.Cleanup());
1029 }
1030
1031 return result;
1032 }
1033 }
1034
1035 internal static AnalysisCacheData Get()
1036 {
1037 int retryCount = 3;
1038
1039 do
1040 {
1041 try
1042 {
1043 if (File.Exists(s_cacheStoreLocation))
1044 {
1045 return Deserialize(s_cacheStoreLocation);
1046 }
1047 }
1048 catch (Exception e)
1049 {
1050 ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache: " + e.Message);
1051 if ((object)e.Message == (object)TruncatedErrorMessage
1052 || (object)e.Message == (object)InvalidSignatureErrorMessage
1053 || (object)e.Message == (object)PossibleCorruptionErrorMessage)
1054 {
1055 // Don't retry if we detected something is wrong with the file
1056 // (as opposed to the file being locked or something else)
1057 break;
1058 }
1059 }
1060
1061 retryCount -= 1;
1062 Thread.Sleep(25); // Sleep a bit to give time for another process to finish writing the cache
1063 } while (retryCount > 0);
1064
1065 return new AnalysisCacheData
1066 {
1067 LastReadTime = DateTime.Now,
1068 // Capacity set to 100 - a bit bigger than the # of modules on a default Win10 client machine
1069 // Concurrency=3 to not create too many locks, contention is unclear, but the old code had a single lock
1070 Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, /*capacity*/100, StringComparer.OrdinalIgnoreCase)
1071 };
1072 }
1073
1074 private AnalysisCacheData()
1075 {
1076 }
1077
1078 private static readonly string s_cacheStoreLocation;
1079
1080 static AnalysisCacheData()
1081 {
1082 // If user defines a custom cache path, then use that.
1083 string userDefinedCachePath = Environment.GetEnvironmentVariable("PSModuleAnalysisCachePath");
1084 if (!string.IsNullOrEmpty(userDefinedCachePath))
1085 {
1086 s_cacheStoreLocation = userDefinedCachePath;
1087 return;
1088 }
1089
1090 string cacheFileName = "ModuleAnalysisCache";
1091
1092 // When multiple copies of pwsh are on the system, they should use their own copy of the cache.
1093 // Append hash of `$PSHOME` to cacheFileName.
1094 string hashString = CRC32Hash.ComputeHash(Utils.DefaultPowerShellAppBase);
1095 cacheFileName = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", cacheFileName, hashString);
1096
1097 if (ExperimentalFeature.EnabledExperimentalFeatureNames.Count > 0)
1098 {
1099 // If any experimental features are enabled, we cannot use the default cache file because those
1100 // features may expose commands that are not available in a regular powershell session, and we
1101 // should not cache those commands in the default cache file because that will result in wrong
1102 // auto-completion suggestions when the default cache file is used in another powershell session.
1103 //
1104 // Here we will generate a cache file name that represent the combination of enabled feature names.
1105 // We first convert enabled feature names to lower case, then we sort the feature names, and then
1106 // compute an CRC32 hash from the sorted feature names. We will use the CRC32 hash to generate the
1107 // cache file name.
1108 int index = 0;
1109 string[] featureNames = new string[ExperimentalFeature.EnabledExperimentalFeatureNames.Count];
1110 foreach (string featureName in ExperimentalFeature.EnabledExperimentalFeatureNames)
1111 {
1112 featureNames[index++] = featureName.ToLowerInvariant();
1113 }
1114
1115 Array.Sort(featureNames);
1116 string allNames = string.Join(Environment.NewLine, featureNames);
1117
1118 // Use CRC32 because it's faster.
1119 // It's very unlikely to get collision from hashing the combinations of enabled features names.
1120 hashString = CRC32Hash.ComputeHash(allNames);
1121 cacheFileName = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", cacheFileName, hashString);
1122 }
1123
1124 s_cacheStoreLocation = Path.Combine(Platform.CacheDirectory, cacheFileName);
1125 }
1126 }
1127
1128 [DebuggerDisplay("ModulePath = {ModulePath}")]
1129 internal class ModuleCacheEntry
1130 {
1131 public DateTime LastWriteTime;
1132 public string ModulePath;
1133 public bool TypesAnalyzed;
1134 public ConcurrentDictionary<string, CommandTypes> Commands;
1135 public ConcurrentDictionary<string, TypeAttributes> Types;
1136 }
1137 }