"Fossies" - the Fresh Open Source Software Archive

Member "PowerShell-7.2.6/src/System.Management.Automation/engine/Modules/AnalysisCache.cs" (11 Aug 2022, 50256 Bytes) of package /linux/misc/PowerShell-7.2.6.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) C# source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. For more information about "AnalysisCache.cs" see the Fossies "Dox" file reference documentation.

    1 // Copyright (c) Microsoft Corporation.
    2 // Licensed under the MIT License.
    3 
    4 using System.Collections;
    5 using System.Collections.Concurrent;
    6 using System.Collections.Generic;
    7 using System.Diagnostics;
    8 using System.Globalization;
    9 using System.IO;
   10 using System.Management.Automation.Internal;
   11 using System.Management.Automation.Language;
   12 using System.Management.Automation.Runspaces;
   13 using System.Text;
   14 using System.Threading;
   15 using System.Threading.Tasks;
   16 
   17 using Microsoft.PowerShell.Commands;
   18 
   19 namespace System.Management.Automation
   20 {
   21     /// <summary>
   22     /// Class to manage the caching of analysis data.
   23     /// For performance, module command caching is flattened after discovery. Many modules have nested
   24     /// modules that can only be resolved at runtime - for example,
   25     /// script modules that declare: $env:PATH += "; $psScriptRoot". When
   26     /// doing initial analysis, we include these in 'ExportedCommands'.
   27     /// Changes to these type of modules will not be re-analyzed, unless the user re-imports the module,
   28     /// or runs Get-Module -List.
   29     /// </summary>
   30     internal static class AnalysisCache
   31     {
   32         private static readonly AnalysisCacheData s_cacheData = AnalysisCacheData.Get();
   33 
   34         // This dictionary shouldn't see much use, so low concurrency and capacity
   35         private static readonly ConcurrentDictionary<string, string> s_modulesBeingAnalyzed =
   36             new(concurrencyLevel: 1, capacity: 2, StringComparer.OrdinalIgnoreCase);
   37 
   38         internal static readonly char[] InvalidCommandNameCharacters = new[]
   39         {
   40             '#', ',', '(', ')', '{', '}', '[', ']', '&', '/', '\\', '$', '^', ';', ':',
   41             '"', '\'', '<', '>', '|', '?', '@', '`', '*', '%', '+', '=', '~'
   42         };
   43 
   44         internal static ConcurrentDictionary<string, CommandTypes> GetExportedCommands(string modulePath, bool testOnly, ExecutionContext context)
   45         {
   46             bool etwEnabled = CommandDiscoveryEventSource.Log.IsEnabled();
   47             if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStart(modulePath);
   48 
   49             DateTime lastWriteTime;
   50             ModuleCacheEntry moduleCacheEntry;
   51             if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
   52             {
   53                 if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);
   54                 return moduleCacheEntry.Commands;
   55             }
   56 
   57             ConcurrentDictionary<string, CommandTypes> result = null;
   58 
   59             if (!testOnly)
   60             {
   61                 var extension = Path.GetExtension(modulePath);
   62                 if (extension.Equals(StringLiterals.PowerShellDataFileExtension, StringComparison.OrdinalIgnoreCase))
   63                 {
   64                     result = AnalyzeManifestModule(modulePath, context, lastWriteTime, etwEnabled);
   65                 }
   66                 else if (extension.Equals(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))
   67                 {
   68                     result = AnalyzeScriptModule(modulePath, context, lastWriteTime);
   69                 }
   70                 else if (extension.Equals(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))
   71                 {
   72                     result = AnalyzeCdxmlModule(modulePath, context, lastWriteTime);
   73                 }
   74                 else if (extension.Equals(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))
   75                 {
   76                     result = AnalyzeDllModule(modulePath, context, lastWriteTime);
   77                 }
   78                 else if (extension.Equals(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))
   79                 {
   80                     result = AnalyzeDllModule(modulePath, context, lastWriteTime);
   81                 }
   82             }
   83 
   84             if (result != null)
   85             {
   86                 s_cacheData.QueueSerialization();
   87                 ModuleIntrinsics.Tracer.WriteLine("Returning {0} exported commands.", result.Count);
   88             }
   89             else
   90             {
   91                 ModuleIntrinsics.Tracer.WriteLine("Returning NULL for exported commands.");
   92             }
   93 
   94             if (etwEnabled) CommandDiscoveryEventSource.Log.GetModuleExportedCommandsStop(modulePath);
   95             return result;
   96         }
   97 
   98         private static ConcurrentDictionary<string, CommandTypes> AnalyzeManifestModule(string modulePath, ExecutionContext context, DateTime lastWriteTime, bool etwEnabled)
   99         {
  100             ConcurrentDictionary<string, CommandTypes> result = null;
  101             try
  102             {
  103                 var moduleManifestProperties = PsUtils.GetModuleManifestProperties(modulePath, PsUtils.FastModuleManifestAnalysisPropertyNames);
  104                 if (moduleManifestProperties != null)
  105                 {
  106                     if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && ModuleIsEditionIncompatible(modulePath, moduleManifestProperties))
  107                     {
  108                         ModuleIntrinsics.Tracer.WriteLine($"Module lies on the Windows System32 legacy module path and is incompatible with current PowerShell edition, skipping module: {modulePath}");
  109                         return null;
  110                     }
  111 
  112                     Version version;
  113                     if (ModuleUtils.IsModuleInVersionSubdirectory(modulePath, out version))
  114                     {
  115                         var versionInManifest = LanguagePrimitives.ConvertTo<Version>(moduleManifestProperties["ModuleVersion"]);
  116                         if (version != versionInManifest)
  117                         {
  118                             ModuleIntrinsics.Tracer.WriteLine("ModuleVersion in manifest does not match versioned module directory, skipping module: {0}", modulePath);
  119                             return null;
  120                         }
  121                     }
  122 
  123                     result = new ConcurrentDictionary<string, CommandTypes>(3, moduleManifestProperties.Count, StringComparer.OrdinalIgnoreCase);
  124 
  125                     var sawWildcard = false;
  126                     var hadCmdlets = AddPsd1EntryToResult(result, moduleManifestProperties["CmdletsToExport"], CommandTypes.Cmdlet, ref sawWildcard);
  127                     var hadFunctions = AddPsd1EntryToResult(result, moduleManifestProperties["FunctionsToExport"], CommandTypes.Function, ref sawWildcard);
  128                     var hadAliases = AddPsd1EntryToResult(result, moduleManifestProperties["AliasesToExport"], CommandTypes.Alias, ref sawWildcard);
  129 
  130                     var analysisSucceeded = hadCmdlets && hadFunctions && hadAliases;
  131 
  132                     if (!analysisSucceeded && !sawWildcard && (hadCmdlets || hadFunctions))
  133                     {
  134                         // If we're missing CmdletsToExport, that might still be OK, but only if we have a script module.
  135                         // Likewise, if we're missing FunctionsToExport, that might be OK, but only if we have a binary module.
  136 
  137                         analysisSucceeded = !CheckModulesTypesInManifestAgainstExportedCommands(moduleManifestProperties, hadCmdlets, hadFunctions, hadAliases);
  138                     }
  139 
  140                     if (analysisSucceeded)
  141                     {
  142                         var moduleCacheEntry = new ModuleCacheEntry
  143                         {
  144                             ModulePath = modulePath,
  145                             LastWriteTime = lastWriteTime,
  146                             Commands = result,
  147                             TypesAnalyzed = false,
  148                             Types = new ConcurrentDictionary<string, TypeAttributes>(1, 8, StringComparer.OrdinalIgnoreCase)
  149                         };
  150                         s_cacheData.Entries[modulePath] = moduleCacheEntry;
  151                     }
  152                     else
  153                     {
  154                         result = null;
  155                     }
  156                 }
  157             }
  158             catch (Exception e)
  159             {
  160                 if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisException(modulePath, e.Message);
  161                 // Ignore the errors, proceed with the usual module analysis
  162                 ModuleIntrinsics.Tracer.WriteLine("Exception on fast-path analysis of module {0}", modulePath);
  163             }
  164 
  165             if (etwEnabled) CommandDiscoveryEventSource.Log.ModuleManifestAnalysisResult(modulePath, result != null);
  166 
  167             return result ?? AnalyzeTheOldWay(modulePath, context, lastWriteTime);
  168         }
  169 
  170         /// <summary>
  171         /// Check if a module is compatible with the current PSEdition given its path and its manifest properties.
  172         /// </summary>
  173         /// <param name="modulePath">The path to the module.</param>
  174         /// <param name="moduleManifestProperties">The properties of the module's manifest.</param>
  175         /// <returns></returns>
  176         internal static bool ModuleIsEditionIncompatible(string modulePath, Hashtable moduleManifestProperties)
  177         {
  178 #if UNIX
  179             return false;
  180 #else
  181             if (!ModuleUtils.IsOnSystem32ModulePath(modulePath))
  182             {
  183                 return false;
  184             }
  185 
  186             if (!moduleManifestProperties.ContainsKey("CompatiblePSEditions"))
  187             {
  188                 return true;
  189             }
  190 
  191             return !Utils.IsPSEditionSupported(LanguagePrimitives.ConvertTo<string[]>(moduleManifestProperties["CompatiblePSEditions"]));
  192 #endif
  193         }
  194 
  195         internal static bool ModuleAnalysisViaGetModuleRequired(object modulePathObj, bool hadCmdlets, bool hadFunctions, bool hadAliases)
  196         {
  197             if (!(modulePathObj is string modulePath))
  198                 return true;
  199 
  200             if (modulePath.EndsWith(StringLiterals.PowerShellModuleFileExtension, StringComparison.OrdinalIgnoreCase))
  201             {
  202                 // A script module can't exactly define cmdlets, but it can import a binary module (as nested), so
  203                 // it can indirectly define cmdlets.  And obviously a script module can define functions and aliases.
  204                 // If we got here, one of those is missing, so analysis is required.
  205                 return true;
  206             }
  207 
  208             if (modulePath.EndsWith(StringLiterals.PowerShellCmdletizationFileExtension, StringComparison.OrdinalIgnoreCase))
  209             {
  210                 // A cdxml module can only define functions and aliases, so if we have both, no more analysis is required.
  211                 return !hadFunctions || !hadAliases;
  212             }
  213 
  214             if (modulePath.EndsWith(StringLiterals.PowerShellILAssemblyExtension, StringComparison.OrdinalIgnoreCase))
  215             {
  216                 // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,
  217                 // more analysis is required. If the module exports aliases, we can't discover that analyzing
  218                 // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.
  219                 return !hadCmdlets;
  220             }
  221 
  222             if (modulePath.EndsWith(StringLiterals.PowerShellILExecutableExtension, StringComparison.OrdinalIgnoreCase))
  223             {
  224                 // A dll just exports cmdlets, so if the manifest doesn't explicitly export any cmdlets,
  225                 // more analysis is required. If the module exports aliases, we can't discover that analyzing
  226                 // the binary, so aliases are always required to be explicit (no wildcards) in the manifest.
  227                 return !hadCmdlets;
  228             }
  229 
  230             // Any other extension (or no extension), just assume the worst and analyze the module
  231             return true;
  232         }
  233 
  234         // Returns true if we need to analyze the manifest module in Get-Module because
  235         // our quick and dirty module manifest analysis is missing something not easily
  236         // discovered.
  237         //
  238         // TODO - psm1 modules are actually easily handled, so if we only saw a psm1 here,
  239         // we should just analyze it and not fall back on Get-Module -List.
  240         private static bool CheckModulesTypesInManifestAgainstExportedCommands(Hashtable moduleManifestProperties, bool hadCmdlets, bool hadFunctions, bool hadAliases)
  241         {
  242             var rootModule = moduleManifestProperties["RootModule"];
  243             if (rootModule != null && ModuleAnalysisViaGetModuleRequired(rootModule, hadCmdlets, hadFunctions, hadAliases))
  244                 return true;
  245 
  246             var moduleToProcess = moduleManifestProperties["ModuleToProcess"];
  247             if (moduleToProcess != null && ModuleAnalysisViaGetModuleRequired(moduleToProcess, hadCmdlets, hadFunctions, hadAliases))
  248                 return true;
  249 
  250             var nestedModules = moduleManifestProperties["NestedModules"];
  251             if (nestedModules != null)
  252             {
  253                 var nestedModule = nestedModules as string;
  254                 if (nestedModule != null)
  255                 {
  256                     return ModuleAnalysisViaGetModuleRequired(nestedModule, hadCmdlets, hadFunctions, hadAliases);
  257                 }
  258 
  259                 if (!(nestedModules is object[] nestedModuleArray))
  260                     return true;
  261 
  262                 foreach (var element in nestedModuleArray)
  263                 {
  264                     if (ModuleAnalysisViaGetModuleRequired(element, hadCmdlets, hadFunctions, hadAliases))
  265                         return true;
  266                 }
  267             }
  268 
  269             return false;
  270         }
  271 
  272         private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, string command, CommandTypes commandTypeToAdd, ref bool sawWildcard)
  273         {
  274             if (WildcardPattern.ContainsWildcardCharacters(command))
  275             {
  276                 sawWildcard = true;
  277                 return false;
  278             }
  279 
  280             // An empty string is one way of saying "no exported commands".
  281             if (command.Length != 0)
  282             {
  283                 CommandTypes commandTypes;
  284                 if (result.TryGetValue(command, out commandTypes))
  285                 {
  286                     commandTypes |= commandTypeToAdd;
  287                 }
  288                 else
  289                 {
  290                     commandTypes = commandTypeToAdd;
  291                 }
  292 
  293                 result[command] = commandTypes;
  294             }
  295 
  296             return true;
  297         }
  298 
  299         private static bool AddPsd1EntryToResult(ConcurrentDictionary<string, CommandTypes> result, object value, CommandTypes commandTypeToAdd, ref bool sawWildcard)
  300         {
  301             string command = value as string;
  302             if (command != null)
  303             {
  304                 return AddPsd1EntryToResult(result, command, commandTypeToAdd, ref sawWildcard);
  305             }
  306 
  307             object[] commands = value as object[];
  308             if (commands != null)
  309             {
  310                 foreach (var o in commands)
  311                 {
  312                     if (!AddPsd1EntryToResult(result, o, commandTypeToAdd, ref sawWildcard))
  313                         return false;
  314                 }
  315 
  316                 // An empty array is still success, that's how a manifest declares that
  317                 // no entries are exported (unlike the lack of an entry, or $null).
  318                 return true;
  319             }
  320 
  321             // Unknown type, let Get-Module -List deal with this manifest
  322             return false;
  323         }
  324 
  325         private static ConcurrentDictionary<string, CommandTypes> AnalyzeScriptModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
  326         {
  327             var scriptAnalysis = ScriptAnalysis.Analyze(modulePath, context);
  328             if (scriptAnalysis == null)
  329             {
  330                 return null;
  331             }
  332 
  333             List<WildcardPattern> scriptAnalysisPatterns = new List<WildcardPattern>();
  334             foreach (string discoveredCommandFilter in scriptAnalysis.DiscoveredCommandFilters)
  335             {
  336                 scriptAnalysisPatterns.Add(new WildcardPattern(discoveredCommandFilter));
  337             }
  338 
  339             var result = new ConcurrentDictionary<string, CommandTypes>(3,
  340                 scriptAnalysis.DiscoveredExports.Count + scriptAnalysis.DiscoveredAliases.Count,
  341                 StringComparer.OrdinalIgnoreCase);
  342 
  343             // Add any directly discovered exports
  344             foreach (var command in scriptAnalysis.DiscoveredExports)
  345             {
  346                 if (SessionStateUtilities.MatchesAnyWildcardPattern(command, scriptAnalysisPatterns, true))
  347                 {
  348                     if (command.IndexOfAny(InvalidCommandNameCharacters) < 0)
  349                     {
  350                         result[command] = CommandTypes.Function;
  351                     }
  352                 }
  353             }
  354 
  355             // Add the discovered aliases
  356             foreach (var pair in scriptAnalysis.DiscoveredAliases)
  357             {
  358                 var commandName = pair.Key;
  359                 // These are already filtered
  360                 if (commandName.IndexOfAny(InvalidCommandNameCharacters) < 0)
  361                 {
  362                     result.AddOrUpdate(commandName, CommandTypes.Alias,
  363                         static (_, existingCommandType) => existingCommandType | CommandTypes.Alias);
  364                 }
  365             }
  366 
  367             // Add any files in PsScriptRoot if it added itself to the path
  368             if (scriptAnalysis.AddsSelfToPath)
  369             {
  370                 string baseDirectory = Path.GetDirectoryName(modulePath);
  371 
  372                 try
  373                 {
  374                     foreach (string item in Directory.EnumerateFiles(baseDirectory, "*.ps1"))
  375                     {
  376                         var command = Path.GetFileNameWithoutExtension(item);
  377                         result.AddOrUpdate(command, CommandTypes.ExternalScript,
  378                             static (_, existingCommandType) => existingCommandType | CommandTypes.ExternalScript);
  379                     }
  380                 }
  381                 catch (UnauthorizedAccessException)
  382                 {
  383                     // Consume this exception here
  384                 }
  385             }
  386 
  387             ConcurrentDictionary<string, TypeAttributes> exportedClasses = new(
  388                 concurrencyLevel: 1,
  389                 capacity: scriptAnalysis.DiscoveredClasses.Count,
  390                 StringComparer.OrdinalIgnoreCase);
  391             foreach (var exportedClass in scriptAnalysis.DiscoveredClasses)
  392             {
  393                 exportedClasses[exportedClass.Name] = exportedClass.TypeAttributes;
  394             }
  395 
  396             var moduleCacheEntry = new ModuleCacheEntry
  397             {
  398                 ModulePath = modulePath,
  399                 LastWriteTime = lastWriteTime,
  400                 Commands = result,
  401                 TypesAnalyzed = true,
  402                 Types = exportedClasses
  403             };
  404             s_cacheData.Entries[modulePath] = moduleCacheEntry;
  405 
  406             return result;
  407         }
  408 
  409         private static ConcurrentDictionary<string, CommandTypes> AnalyzeCdxmlModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
  410         {
  411             return AnalyzeTheOldWay(modulePath, context, lastWriteTime);
  412         }
  413 
  414         private static ConcurrentDictionary<string, CommandTypes> AnalyzeDllModule(string modulePath, ExecutionContext context, DateTime lastWriteTime)
  415         {
  416             return AnalyzeTheOldWay(modulePath, context, lastWriteTime);
  417         }
  418 
  419         private static ConcurrentDictionary<string, CommandTypes> AnalyzeTheOldWay(string modulePath, ExecutionContext context, DateTime lastWriteTime)
  420         {
  421             try
  422             {
  423                 // If we're already analyzing this module, let the recursion bottom out.
  424                 if (!s_modulesBeingAnalyzed.TryAdd(modulePath, modulePath))
  425                 {
  426                     ModuleIntrinsics.Tracer.WriteLine("{0} is already being analyzed. Exiting.", modulePath);
  427                     return null;
  428                 }
  429 
  430                 // Record that we're analyzing this specific module so that we don't get stuck in recursion
  431                 ModuleIntrinsics.Tracer.WriteLine("Started analysis: {0}", modulePath);
  432                 CallGetModuleDashList(context, modulePath);
  433 
  434                 ModuleCacheEntry moduleCacheEntry;
  435                 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
  436                 {
  437                     return moduleCacheEntry.Commands;
  438                 }
  439             }
  440             catch (Exception e)
  441             {
  442                 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
  443 
  444                 // Catch-all OK, third-party call-out.
  445             }
  446             finally
  447             {
  448                 ModuleIntrinsics.Tracer.WriteLine("Finished analysis: {0}", modulePath);
  449                 s_modulesBeingAnalyzed.TryRemove(modulePath, out modulePath);
  450             }
  451 
  452             return null;
  453         }
  454 
  455         /// <summary>
  456         /// Return the exported types for a specific module.
  457         /// If the module is already cache, return from cache, else cache the module.
  458         /// Also re-cache the module if the cached item is stale.
  459         /// </summary>
  460         /// <param name="modulePath">Path to the module to get exported types from.</param>
  461         /// <param name="context">Current Context.</param>
  462         /// <returns></returns>
  463         internal static ConcurrentDictionary<string, TypeAttributes> GetExportedClasses(string modulePath, ExecutionContext context)
  464         {
  465             DateTime lastWriteTime;
  466             ModuleCacheEntry moduleCacheEntry;
  467             if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry) && moduleCacheEntry.TypesAnalyzed)
  468             {
  469                 return moduleCacheEntry.Types;
  470             }
  471 
  472             try
  473             {
  474                 CallGetModuleDashList(context, modulePath);
  475                 if (GetModuleEntryFromCache(modulePath, out lastWriteTime, out moduleCacheEntry))
  476                 {
  477                     return moduleCacheEntry.Types;
  478                 }
  479             }
  480             catch (Exception e)
  481             {
  482                 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
  483 
  484                 // Catch-all OK, third-party call-out.
  485             }
  486 
  487             return null;
  488         }
  489 
  490         internal static void CacheModuleExports(PSModuleInfo module, ExecutionContext context)
  491         {
  492             ModuleIntrinsics.Tracer.WriteLine("Requested caching for {0}", module.Name);
  493 
  494             // Don't cache incompatible modules on the system32 module path even if loaded with
  495             // -SkipEditionCheck, since it will break subsequent sessions
  496             if (!Configuration.PowerShellConfig.Instance.IsImplicitWinCompatEnabled() && !module.IsConsideredEditionCompatible)
  497             {
  498                 ModuleIntrinsics.Tracer.WriteLine($"Module '{module.Name}' not edition compatible and not cached.");
  499                 return;
  500             }
  501 
  502             DateTime lastWriteTime;
  503             ModuleCacheEntry moduleCacheEntry;
  504             GetModuleEntryFromCache(module.Path, out lastWriteTime, out moduleCacheEntry);
  505 
  506             var realExportedCommands = module.ExportedCommands;
  507             var realExportedClasses = module.GetExportedTypeDefinitions();
  508             ConcurrentDictionary<string, CommandTypes> exportedCommands;
  509             ConcurrentDictionary<string, TypeAttributes> exportedClasses;
  510 
  511             // First see if the existing module info is sufficient. GetModuleEntryFromCache does LastWriteTime
  512             // verification, so this will also return nothing if the cache is out of date or corrupt.
  513             if (moduleCacheEntry != null)
  514             {
  515                 bool needToUpdate = false;
  516 
  517                 // We need to iterate and check as exportedCommands will have more item as it can have aliases as well.
  518                 exportedCommands = moduleCacheEntry.Commands;
  519                 foreach (var pair in realExportedCommands)
  520                 {
  521                     var commandName = pair.Key;
  522                     var realCommandType = pair.Value.CommandType;
  523                     CommandTypes commandType;
  524                     if (!exportedCommands.TryGetValue(commandName, out commandType) || commandType != realCommandType)
  525                     {
  526                         needToUpdate = true;
  527                         break;
  528                     }
  529                 }
  530 
  531                 exportedClasses = moduleCacheEntry.Types;
  532                 foreach (var pair in realExportedClasses)
  533                 {
  534                     var className = pair.Key;
  535                     var realTypeAttributes = pair.Value.TypeAttributes;
  536                     TypeAttributes typeAttributes;
  537                     if (!exportedClasses.TryGetValue(className, out typeAttributes) ||
  538                         typeAttributes != realTypeAttributes)
  539                     {
  540                         needToUpdate = true;
  541                         break;
  542                     }
  543                 }
  544 
  545                 // Update or not, we've analyzed commands and types now.
  546                 moduleCacheEntry.TypesAnalyzed = true;
  547 
  548                 if (!needToUpdate)
  549                 {
  550                     ModuleIntrinsics.Tracer.WriteLine("Existing cached info up-to-date. Skipping.");
  551                     return;
  552                 }
  553 
  554                 exportedCommands.Clear();
  555                 exportedClasses.Clear();
  556             }
  557             else
  558             {
  559                 exportedCommands = new ConcurrentDictionary<string, CommandTypes>(3, realExportedCommands.Count, StringComparer.OrdinalIgnoreCase);
  560                 exportedClasses = new ConcurrentDictionary<string, TypeAttributes>(1, realExportedClasses.Count, StringComparer.OrdinalIgnoreCase);
  561                 moduleCacheEntry = new ModuleCacheEntry
  562                 {
  563                     ModulePath = module.Path,
  564                     LastWriteTime = lastWriteTime,
  565                     Commands = exportedCommands,
  566                     TypesAnalyzed = true,
  567                     Types = exportedClasses
  568                 };
  569                 moduleCacheEntry = s_cacheData.Entries.GetOrAdd(module.Path, moduleCacheEntry);
  570             }
  571 
  572             // We need to update the cache
  573             foreach (var exportedCommand in realExportedCommands.Values)
  574             {
  575                 ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", exportedCommand.Name);
  576                 exportedCommands.GetOrAdd(exportedCommand.Name, exportedCommand.CommandType);
  577             }
  578 
  579             foreach (var pair in realExportedClasses)
  580             {
  581                 var className = pair.Key;
  582                 ModuleIntrinsics.Tracer.WriteLine("Caching command: {0}", className);
  583                 moduleCacheEntry.Types.AddOrUpdate(className, pair.Value.TypeAttributes, (k, t) => t);
  584             }
  585 
  586             s_cacheData.QueueSerialization();
  587         }
  588 
  589         private static void CallGetModuleDashList(ExecutionContext context, string modulePath)
  590         {
  591             CommandInfo commandInfo = new CmdletInfo("Get-Module", typeof(GetModuleCommand), null, null, context);
  592             Command getModuleCommand = new Command(commandInfo);
  593 
  594             try
  595             {
  596                 PowerShell.Create(RunspaceMode.CurrentRunspace)
  597                     .AddCommand(getModuleCommand)
  598                         .AddParameter("List", true)
  599                         .AddParameter("ErrorAction", ActionPreference.Ignore)
  600                         .AddParameter("WarningAction", ActionPreference.Ignore)
  601                         .AddParameter("InformationAction", ActionPreference.Ignore)
  602                         .AddParameter("Verbose", false)
  603                         .AddParameter("Debug", false)
  604                         .AddParameter("Name", modulePath)
  605                     .Invoke();
  606             }
  607             catch (Exception e)
  608             {
  609                 ModuleIntrinsics.Tracer.WriteLine("Module analysis generated an exception: {0}", e);
  610 
  611                 // Catch-all OK, third-party call-out.
  612             }
  613         }
  614 
  615         private static bool GetModuleEntryFromCache(string modulePath, out DateTime lastWriteTime, out ModuleCacheEntry moduleCacheEntry)
  616         {
  617             try
  618             {
  619                 lastWriteTime = new FileInfo(modulePath).LastWriteTime;
  620             }
  621             catch (Exception e)
  622             {
  623                 ModuleIntrinsics.Tracer.WriteLine("Exception checking LastWriteTime on module {0}: {1}", modulePath, e.Message);
  624                 lastWriteTime = DateTime.MinValue;
  625             }
  626 
  627             if (s_cacheData.Entries.TryGetValue(modulePath, out moduleCacheEntry))
  628             {
  629                 if (lastWriteTime == moduleCacheEntry.LastWriteTime)
  630                 {
  631                     return true;
  632                 }
  633 
  634                 ModuleIntrinsics.Tracer.WriteLine("{0}: cache entry out of date, cached on {1}, last updated on {2}",
  635                     modulePath, moduleCacheEntry.LastWriteTime, lastWriteTime);
  636 
  637                 s_cacheData.Entries.TryRemove(modulePath, out moduleCacheEntry);
  638             }
  639 
  640             moduleCacheEntry = null;
  641             return false;
  642         }
  643     }
  644 
  645     internal sealed class AnalysisCacheData
  646     {
  647         private static byte[] GetHeader()
  648         {
  649             return new byte[]
  650             {
  651                 0x50, 0x53, 0x4d, 0x4f, 0x44, 0x55, 0x4c, 0x45, 0x43, 0x41, 0x43, 0x48, 0x45, // PSMODULECACHE
  652                 0x01 // version #
  653             };
  654         }
  655 
  656         // The last time the index was maintained.
  657         public DateTime LastReadTime { get; set; }
  658 
  659         public ConcurrentDictionary<string, ModuleCacheEntry> Entries { get; set; }
  660 
  661         private int _saveCacheToDiskQueued;
  662 
  663         private bool _saveCacheToDisk = true;
  664 
  665         public void QueueSerialization()
  666         {
  667             // We expect many modules to rapidly call for serialization.
  668             // Instead of doing it right away, we'll queue a task that starts writing
  669             // after it seems like we've stopped adding stuff to write out.  This is
  670             // avoids blocking the pipeline thread waiting for the write to finish.
  671             // We want to make sure we only queue one task.
  672             if (_saveCacheToDisk && Interlocked.Increment(ref _saveCacheToDiskQueued) == 1)
  673             {
  674                 Task.Run(async delegate
  675                 {
  676                     // Wait a while before assuming we've finished the updates,
  677                     // writing the cache out in a timely matter isn't too important
  678                     // now anyway.
  679                     await Task.Delay(10000).ConfigureAwait(false);
  680                     int counter1, counter2;
  681                     do
  682                     {
  683                         // Check the counter a couple times with a delay,
  684                         // if it's stable, then proceed with writing.
  685                         counter1 = _saveCacheToDiskQueued;
  686                         await Task.Delay(3000).ConfigureAwait(false);
  687                         counter2 = _saveCacheToDiskQueued;
  688                     } while (counter1 != counter2);
  689                     Serialize(s_cacheStoreLocation);
  690                 });
  691             }
  692         }
  693 
  694         // Remove entries that are not needed anymore, e.g. if a module was removed.
  695         // If anything is removed, save the cache.
  696         private void Cleanup()
  697         {
  698             Diagnostics.Assert(Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null,
  699                 "Caller to check environment variable before calling");
  700 
  701             bool removedSomething = false;
  702             var keys = Entries.Keys;
  703             foreach (var key in keys)
  704             {
  705                 if (!File.Exists(key))
  706                 {
  707                     removedSomething |= Entries.TryRemove(key, out ModuleCacheEntry _);
  708                 }
  709             }
  710 
  711             if (removedSomething)
  712             {
  713                 QueueSerialization();
  714             }
  715         }
  716 
  717         private static unsafe void Write(int val, byte[] bytes, FileStream stream)
  718         {
  719             Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");
  720             fixed (byte* b = bytes) *((int*)b) = val;
  721             stream.Write(bytes, 0, 4);
  722         }
  723 
  724         private static unsafe void Write(long val, byte[] bytes, FileStream stream)
  725         {
  726             Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");
  727             fixed (byte* b = bytes) *((long*)b) = val;
  728             stream.Write(bytes, 0, 8);
  729         }
  730 
  731         private static void Write(string val, byte[] bytes, FileStream stream)
  732         {
  733             Write(val.Length, bytes, stream);
  734             bytes = Encoding.UTF8.GetBytes(val);
  735             stream.Write(bytes, 0, bytes.Length);
  736         }
  737 
  738         private void Serialize(string filename)
  739         {
  740             AnalysisCacheData fromOtherProcess = null;
  741             Diagnostics.Assert(_saveCacheToDisk, "Serialize should never be called without going through QueueSerialization which has a check");
  742 
  743             try
  744             {
  745                 if (File.Exists(filename))
  746                 {
  747                     var fileLastWriteTime = new FileInfo(filename).LastWriteTime;
  748                     if (fileLastWriteTime > this.LastReadTime)
  749                     {
  750                         fromOtherProcess = Deserialize(filename);
  751                     }
  752                 }
  753                 else
  754                 {
  755                     // Make sure the folder exists
  756                     var folder = Path.GetDirectoryName(filename);
  757                     if (!Directory.Exists(folder))
  758                     {
  759                         try
  760                         {
  761                             Directory.CreateDirectory(folder);
  762                         }
  763                         catch (UnauthorizedAccessException)
  764                         {
  765                             // service accounts won't be able to create directory
  766                             _saveCacheToDisk = false;
  767                             return;
  768                         }
  769                     }
  770                 }
  771             }
  772             catch (Exception e)
  773             {
  774                 ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache {0}: {1} ", filename, e.Message);
  775             }
  776 
  777             if (fromOtherProcess != null)
  778             {
  779                 // We should merge with what another process wrote so we don't clobber useful analysis
  780                 foreach (var otherEntryPair in fromOtherProcess.Entries)
  781                 {
  782                     var otherModuleName = otherEntryPair.Key;
  783                     var otherEntry = otherEntryPair.Value;
  784                     ModuleCacheEntry thisEntry;
  785                     if (Entries.TryGetValue(otherModuleName, out thisEntry))
  786                     {
  787                         if (otherEntry.LastWriteTime > thisEntry.LastWriteTime)
  788                         {
  789                             // The other entry is newer, take it over ours
  790                             Entries[otherModuleName] = otherEntry;
  791                         }
  792                     }
  793                     else
  794                     {
  795                         Entries[otherModuleName] = otherEntry;
  796                     }
  797                 }
  798             }
  799 
  800             // "PSMODULECACHE"     -> 13 bytes
  801             // byte     ( 1 byte)  -> version
  802             // int      ( 4 bytes) -> count of entries
  803             // entries  (?? bytes) -> all entries
  804             //
  805             // each entry is
  806             //   DateTime ( 8 bytes) -> last write time for module file
  807             //   int      ( 4 bytes) -> path length
  808             //   string   (?? bytes) -> utf8 encoded path
  809             //   int      ( 4 bytes) -> count of commands
  810             //   commands (?? bytes) -> all commands
  811             //   int      ( 4 bytes) -> count of types, -1 means unanalyzed (and 0 items serialized)
  812             //   types    (?? bytes) -> all types
  813             //
  814             // each command is
  815             //   int      ( 4 bytes) -> command name length
  816             //   string   (?? bytes) -> utf8 encoded command name
  817             //   int      ( 4 bytes) -> CommandTypes enum
  818             //
  819             // each type is
  820             //   int     ( 4 bytes) -> type name length
  821             //   string  (?? bytes) -> utf8 encoded type name
  822             //   int     ( 4 bytes) -> type attributes
  823             try
  824             {
  825                 var bytes = new byte[8];
  826 
  827                 using (var stream = File.Create(filename))
  828                 {
  829                     var headerBytes = GetHeader();
  830                     stream.Write(headerBytes, 0, headerBytes.Length);
  831 
  832                     // Count of entries
  833                     Write(Entries.Count, bytes, stream);
  834 
  835                     foreach (var pair in Entries.ToArray())
  836                     {
  837                         var path = pair.Key;
  838                         var entry = pair.Value;
  839 
  840                         // Module last write time
  841                         Write(entry.LastWriteTime.Ticks, bytes, stream);
  842 
  843                         // Module path
  844                         Write(path, bytes, stream);
  845 
  846                         // Commands
  847                         var commandPairs = entry.Commands.ToArray();
  848                         Write(commandPairs.Length, bytes, stream);
  849 
  850                         foreach (var command in commandPairs)
  851                         {
  852                             Write(command.Key, bytes, stream);
  853                             Write((int)command.Value, bytes, stream);
  854                         }
  855 
  856                         // Types
  857                         var typePairs = entry.Types.ToArray();
  858                         Write(entry.TypesAnalyzed ? typePairs.Length : -1, bytes, stream);
  859 
  860                         foreach (var type in typePairs)
  861                         {
  862                             Write(type.Key, bytes, stream);
  863                             Write((int)type.Value, bytes, stream);
  864                         }
  865                     }
  866                 }
  867                 // We just wrote the file, note this so we can detect writes from another process
  868                 LastReadTime = new FileInfo(filename).LastWriteTime;
  869             }
  870             catch (Exception e)
  871             {
  872                 ModuleIntrinsics.Tracer.WriteLine("Exception writing module analysis cache {0}: {1} ", filename, e.Message);
  873             }
  874 
  875             // Reset our counter so we can write again if asked.
  876             Interlocked.Exchange(ref _saveCacheToDiskQueued, 0);
  877         }
  878 
  879         private const string TruncatedErrorMessage = "module cache file appears truncated";
  880         private const string InvalidSignatureErrorMessage = "module cache signature not valid";
  881         private const string PossibleCorruptionErrorMessage = "possible corruption in module cache";
  882 
  883         private static unsafe long ReadLong(FileStream stream, byte[] bytes)
  884         {
  885             Diagnostics.Assert(bytes.Length >= 8, "Must pass a large enough byte array");
  886             if (stream.Read(bytes, 0, 8) != 8)
  887                 throw new Exception(TruncatedErrorMessage);
  888             fixed (byte* b = bytes)
  889                 return *(long*)b;
  890         }
  891 
  892         private static unsafe int ReadInt(FileStream stream, byte[] bytes)
  893         {
  894             Diagnostics.Assert(bytes.Length >= 4, "Must pass a large enough byte array");
  895             if (stream.Read(bytes, 0, 4) != 4)
  896                 throw new Exception(TruncatedErrorMessage);
  897             fixed (byte* b = bytes)
  898                 return *(int*)b;
  899         }
  900 
  901         private static string ReadString(FileStream stream, ref byte[] bytes)
  902         {
  903             int length = ReadInt(stream, bytes);
  904             if (length > 10 * 1024)
  905                 throw new Exception(PossibleCorruptionErrorMessage);
  906             if (length > bytes.Length)
  907                 bytes = new byte[length];
  908             if (stream.Read(bytes, 0, length) != length)
  909                 throw new Exception(TruncatedErrorMessage);
  910             return Encoding.UTF8.GetString(bytes, 0, length);
  911         }
  912 
  913         private static void ReadHeader(FileStream stream, byte[] bytes)
  914         {
  915             var headerBytes = GetHeader();
  916             var length = headerBytes.Length;
  917             Diagnostics.Assert(bytes.Length >= length, "must pass a large enough byte array");
  918             if (stream.Read(bytes, 0, length) != length)
  919                 throw new Exception(TruncatedErrorMessage);
  920 
  921             for (int i = 0; i < length; i++)
  922             {
  923                 if (bytes[i] != headerBytes[i])
  924                 {
  925                     throw new Exception(InvalidSignatureErrorMessage);
  926                 }
  927             }
  928             // No need to return - we don't use it other than to detect the correct file format
  929         }
  930 
  931         public static AnalysisCacheData Deserialize(string filename)
  932         {
  933             using (var stream = File.OpenRead(filename))
  934             {
  935                 var result = new AnalysisCacheData { LastReadTime = DateTime.Now };
  936 
  937                 var bytes = new byte[1024];
  938 
  939                 // Header
  940                 // "PSMODULECACHE"     -> 13 bytes
  941                 // byte     ( 1 byte)  -> version
  942                 ReadHeader(stream, bytes);
  943 
  944                 // int      ( 4 bytes) -> count of entries
  945                 int entries = ReadInt(stream, bytes);
  946                 if (entries > 20 * 1024)
  947                     throw new Exception(PossibleCorruptionErrorMessage);
  948 
  949                 result.Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, entries, StringComparer.OrdinalIgnoreCase);
  950 
  951                 // entries  (?? bytes) -> all entries
  952                 while (entries > 0)
  953                 {
  954                     //   DateTime ( 8 bytes) -> last write time for module file
  955                     var lastWriteTime = new DateTime(ReadLong(stream, bytes));
  956 
  957                     //   int      ( 4 bytes) -> path length
  958                     //   string   (?? bytes) -> utf8 encoded path
  959                     var path = ReadString(stream, ref bytes);
  960 
  961                     //   int      ( 4 bytes) -> count of commands
  962                     var countItems = ReadInt(stream, bytes);
  963                     if (countItems > 20 * 1024)
  964                         throw new Exception(PossibleCorruptionErrorMessage);
  965 
  966                     var commands = new ConcurrentDictionary<string, CommandTypes>(/*concurrency*/3, countItems, StringComparer.OrdinalIgnoreCase);
  967 
  968                     //   commands (?? bytes) -> all commands
  969                     while (countItems > 0)
  970                     {
  971                         //   int      ( 4 bytes) -> command name length
  972                         //   string   (?? bytes) -> utf8 encoded command name
  973                         var commandName = ReadString(stream, ref bytes);
  974 
  975                         //   int      ( 4 bytes) -> CommandTypes enum
  976                         var commandTypes = (CommandTypes)ReadInt(stream, bytes);
  977 
  978                         // Ignore empty entries (possible corruption in the cache or bug?)
  979                         if (!string.IsNullOrWhiteSpace(commandName))
  980                             commands[commandName] = commandTypes;
  981 
  982                         countItems -= 1;
  983                     }
  984 
  985                     //   int      ( 4 bytes) -> count of types
  986                     countItems = ReadInt(stream, bytes);
  987 
  988                     bool typesAnalyzed = countItems != -1;
  989                     if (!typesAnalyzed)
  990                         countItems = 0;
  991                     if (countItems > 20 * 1024)
  992                         throw new Exception(PossibleCorruptionErrorMessage);
  993 
  994                     var types = new ConcurrentDictionary<string, TypeAttributes>(1, countItems, StringComparer.OrdinalIgnoreCase);
  995 
  996                     //   types    (?? bytes) -> all types
  997                     while (countItems > 0)
  998                     {
  999                         //   int     ( 4 bytes) -> type name length
 1000                         //   string  (?? bytes) -> utf8 encoded type name
 1001                         var typeName = ReadString(stream, ref bytes);
 1002 
 1003                         //   int     ( 4 bytes) -> type attributes
 1004                         var typeAttributes = (TypeAttributes)ReadInt(stream, bytes);
 1005 
 1006                         // Ignore empty entries (possible corruption in the cache or bug?)
 1007                         if (!string.IsNullOrWhiteSpace(typeName))
 1008                             types[typeName] = typeAttributes;
 1009 
 1010                         countItems -= 1;
 1011                     }
 1012 
 1013                     var entry = new ModuleCacheEntry
 1014                     {
 1015                         ModulePath = path,
 1016                         LastWriteTime = lastWriteTime,
 1017                         Commands = commands,
 1018                         TypesAnalyzed = typesAnalyzed,
 1019                         Types = types
 1020                     };
 1021                     result.Entries[path] = entry;
 1022 
 1023                     entries -= 1;
 1024                 }
 1025 
 1026                 if (Environment.GetEnvironmentVariable("PSDisableModuleAnalysisCacheCleanup") == null)
 1027                 {
 1028                     Task.Delay(10000).ContinueWith(_ => result.Cleanup());
 1029                 }
 1030 
 1031                 return result;
 1032             }
 1033         }
 1034 
 1035         internal static AnalysisCacheData Get()
 1036         {
 1037             int retryCount = 3;
 1038 
 1039             do
 1040             {
 1041                 try
 1042                 {
 1043                     if (File.Exists(s_cacheStoreLocation))
 1044                     {
 1045                         return Deserialize(s_cacheStoreLocation);
 1046                     }
 1047                 }
 1048                 catch (Exception e)
 1049                 {
 1050                     ModuleIntrinsics.Tracer.WriteLine("Exception checking module analysis cache: " + e.Message);
 1051                     if ((object)e.Message == (object)TruncatedErrorMessage
 1052                         || (object)e.Message == (object)InvalidSignatureErrorMessage
 1053                         || (object)e.Message == (object)PossibleCorruptionErrorMessage)
 1054                     {
 1055                         // Don't retry if we detected something is wrong with the file
 1056                         // (as opposed to the file being locked or something else)
 1057                         break;
 1058                     }
 1059                 }
 1060 
 1061                 retryCount -= 1;
 1062                 Thread.Sleep(25); // Sleep a bit to give time for another process to finish writing the cache
 1063             } while (retryCount > 0);
 1064 
 1065             return new AnalysisCacheData
 1066             {
 1067                 LastReadTime = DateTime.Now,
 1068                 // Capacity set to 100 - a bit bigger than the # of modules on a default Win10 client machine
 1069                 // Concurrency=3 to not create too many locks, contention is unclear, but the old code had a single lock
 1070                 Entries = new ConcurrentDictionary<string, ModuleCacheEntry>(/*concurrency*/3, /*capacity*/100, StringComparer.OrdinalIgnoreCase)
 1071             };
 1072         }
 1073 
 1074         private AnalysisCacheData()
 1075         {
 1076         }
 1077 
 1078         private static readonly string s_cacheStoreLocation;
 1079 
 1080         static AnalysisCacheData()
 1081         {
 1082             // If user defines a custom cache path, then use that.
 1083             string userDefinedCachePath = Environment.GetEnvironmentVariable("PSModuleAnalysisCachePath");
 1084             if (!string.IsNullOrEmpty(userDefinedCachePath))
 1085             {
 1086                 s_cacheStoreLocation = userDefinedCachePath;
 1087                 return;
 1088             }
 1089 
 1090             string cacheFileName = "ModuleAnalysisCache";
 1091 
 1092             // When multiple copies of pwsh are on the system, they should use their own copy of the cache.
 1093             // Append hash of `$PSHOME` to cacheFileName.
 1094             string hashString = CRC32Hash.ComputeHash(Utils.DefaultPowerShellAppBase);
 1095             cacheFileName = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", cacheFileName, hashString);
 1096 
 1097             if (ExperimentalFeature.EnabledExperimentalFeatureNames.Count > 0)
 1098             {
 1099                 // If any experimental features are enabled, we cannot use the default cache file because those
 1100                 // features may expose commands that are not available in a regular powershell session, and we
 1101                 // should not cache those commands in the default cache file because that will result in wrong
 1102                 // auto-completion suggestions when the default cache file is used in another powershell session.
 1103                 //
 1104                 // Here we will generate a cache file name that represent the combination of enabled feature names.
 1105                 // We first convert enabled feature names to lower case, then we sort the feature names, and then
 1106                 // compute an CRC32 hash from the sorted feature names. We will use the CRC32 hash to generate the
 1107                 // cache file name.
 1108                 int index = 0;
 1109                 string[] featureNames = new string[ExperimentalFeature.EnabledExperimentalFeatureNames.Count];
 1110                 foreach (string featureName in ExperimentalFeature.EnabledExperimentalFeatureNames)
 1111                 {
 1112                     featureNames[index++] = featureName.ToLowerInvariant();
 1113                 }
 1114 
 1115                 Array.Sort(featureNames);
 1116                 string allNames = string.Join(Environment.NewLine, featureNames);
 1117 
 1118                 // Use CRC32 because it's faster.
 1119                 // It's very unlikely to get collision from hashing the combinations of enabled features names.
 1120                 hashString = CRC32Hash.ComputeHash(allNames);
 1121                 cacheFileName = string.Format(CultureInfo.InvariantCulture, "{0}-{1}", cacheFileName, hashString);
 1122             }
 1123 
 1124             s_cacheStoreLocation = Path.Combine(Platform.CacheDirectory, cacheFileName);
 1125         }
 1126     }
 1127 
 1128     [DebuggerDisplay("ModulePath = {ModulePath}")]
 1129     internal class ModuleCacheEntry
 1130     {
 1131         public DateTime LastWriteTime;
 1132         public string ModulePath;
 1133         public bool TypesAnalyzed;
 1134         public ConcurrentDictionary<string, CommandTypes> Commands;
 1135         public ConcurrentDictionary<string, TypeAttributes> Types;
 1136     }
 1137 }