"Fossies" - the Fresh Open Source Software Archive

Member "cli-1.1280.1/packages/snyk-fix/src/plugins/python/handlers/pip-requirements/index.ts" (20 Feb 2024, 9414 Bytes) of package /linux/misc/snyk-cli-1.1280.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) TypeScript source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 import * as debugLib from 'debug';
    2 import * as pathLib from 'path';
    3 const sortBy = require('lodash.sortby');
    4 const groupBy = require('lodash.groupby');
    5 
    6 import {
    7   EntityToFix,
    8   FixChangesSummary,
    9   FixOptions,
   10   Issue,
   11   RemediationChanges,
   12   Workspace,
   13 } from '../../../../types';
   14 import { FixedCache, PluginFixResponse } from '../../../types';
   15 import { updateDependencies } from './update-dependencies';
   16 import { NoFixesCouldBeAppliedError } from '../../../../lib/errors/no-fixes-applied';
   17 import { extractProvenance } from './extract-version-provenance';
   18 import {
   19   ParsedRequirements,
   20   parseRequirementsFile,
   21 } from './update-dependencies/requirements-file-parser';
   22 import { standardizePackageName } from '../../standardize-package-name';
   23 import { containsRequireDirective } from './contains-require-directive';
   24 import { validateRequiredData } from '../validate-required-data';
   25 import { formatDisplayName } from '../../../../lib/output-formatters/format-display-name';
   26 
   27 const debug = debugLib('snyk-fix:python:requirements.txt');
   28 
   29 export async function pipRequirementsTxt(
   30   fixable: EntityToFix[],
   31   options: FixOptions,
   32 ): Promise<PluginFixResponse> {
   33   debug(`Preparing to fix ${fixable.length} Python requirements.txt projects`);
   34   const handlerResult: PluginFixResponse = {
   35     succeeded: [],
   36     failed: [],
   37     skipped: [],
   38   };
   39 
   40   const ordered = sortByDirectory(fixable);
   41   let fixedFilesCache: FixedCache = {};
   42   for (const dir of Object.keys(ordered)) {
   43     debug(`Fixing entities in directory ${dir}`);
   44     const entitiesPerDirectory = ordered[dir].map((e) => e.entity);
   45     const { failed, succeeded, skipped, fixedCache } = await fixAll(
   46       entitiesPerDirectory,
   47       options,
   48       fixedFilesCache,
   49     );
   50     fixedFilesCache = {
   51       ...fixedFilesCache,
   52       ...fixedCache,
   53     };
   54     handlerResult.succeeded.push(...succeeded);
   55     handlerResult.failed.push(...failed);
   56     handlerResult.skipped.push(...skipped);
   57   }
   58   return handlerResult;
   59 }
   60 
   61 async function fixAll(
   62   entities: EntityToFix[],
   63   options: FixOptions,
   64   fixedCache: FixedCache,
   65 ): Promise<PluginFixResponse & { fixedCache: FixedCache }> {
   66   const handlerResult: PluginFixResponse = {
   67     succeeded: [],
   68     failed: [],
   69     skipped: [],
   70   };
   71   for (const entity of entities) {
   72     const targetFile = entity.scanResult.identity.targetFile!;
   73     try {
   74       const { dir, base } = pathLib.parse(targetFile);
   75       // parse & join again to support correct separator
   76       const filePath = pathLib.normalize(pathLib.join(dir, base));
   77       if (
   78         Object.keys(fixedCache).includes(
   79           pathLib.normalize(pathLib.join(dir, base)),
   80         )
   81       ) {
   82         handlerResult.succeeded.push({
   83           original: entity,
   84           changes: [
   85             {
   86               success: true,
   87               userMessage: `Fixed through ${formatDisplayName(
   88                 entity.workspace.path,
   89                 {
   90                   type: entity.scanResult.identity.type,
   91                   targetFile: fixedCache[filePath].fixedIn,
   92                 },
   93               )}`,
   94               issueIds: getFixedEntityIssues(
   95                 fixedCache[filePath].issueIds,
   96                 entity.testResult.issues,
   97               ),
   98             },
   99           ],
  100         });
  101         continue;
  102       }
  103       const { changes, fixedMeta } = await applyAllFixes(entity, options);
  104       if (!changes.length) {
  105         debug('Manifest has not changed!');
  106         throw new NoFixesCouldBeAppliedError();
  107       }
  108 
  109       // keep issues were successfully fixed unique across files that are part of the same project
  110       // the test result is for 1 entry entity.
  111       const uniqueIssueIds = new Set<string>();
  112       for (const c of changes) {
  113         c.issueIds.map((i) => uniqueIssueIds.add(i));
  114       }
  115       Object.keys(fixedMeta).forEach((f) => {
  116         fixedCache[f] = {
  117           fixedIn: targetFile,
  118           issueIds: Array.from(uniqueIssueIds),
  119         };
  120       });
  121       handlerResult.succeeded.push({ original: entity, changes });
  122     } catch (e) {
  123       debug(`Failed to fix ${targetFile}.\nERROR: ${e}`);
  124       handlerResult.failed.push({ original: entity, error: e });
  125     }
  126   }
  127   return { ...handlerResult, fixedCache };
  128 }
  129 
  130 // TODO: optionally verify the deps install
  131 export async function fixIndividualRequirementsTxt(
  132   workspace: Workspace,
  133   dir: string,
  134   entryFileName: string,
  135   fileName: string,
  136   remediation: RemediationChanges,
  137   parsedRequirements: ParsedRequirements,
  138   options: FixOptions,
  139   directUpgradesOnly: boolean,
  140 ): Promise<{ changes: FixChangesSummary[] }> {
  141   const entryFilePath = pathLib.normalize(pathLib.join(dir, entryFileName));
  142   const fullFilePath = pathLib.normalize(pathLib.join(dir, fileName));
  143   const { updatedManifest, changes } = updateDependencies(
  144     parsedRequirements,
  145     remediation.pin,
  146     directUpgradesOnly,
  147     entryFilePath !== fullFilePath
  148       ? formatDisplayName(workspace.path, {
  149           type: 'pip',
  150           targetFile: fullFilePath,
  151         })
  152       : undefined,
  153   );
  154 
  155   if (!changes.length) {
  156     return { changes };
  157   }
  158 
  159   if (!options.dryRun) {
  160     debug('Writing changes to file');
  161     await workspace.writeFile(pathLib.join(dir, fileName), updatedManifest);
  162   } else {
  163     debug('Skipping writing changes to file in --dry-run mode');
  164   }
  165 
  166   return { changes };
  167 }
  168 
  169 export async function applyAllFixes(
  170   entity: EntityToFix,
  171   options: FixOptions,
  172 ): Promise<{
  173   changes: FixChangesSummary[];
  174   fixedMeta: { [filePath: string]: FixChangesSummary[] };
  175 }> {
  176   const {
  177     remediation,
  178     targetFile: entryFileName,
  179     workspace,
  180   } = validateRequiredData(entity);
  181   const fixedMeta: {
  182     [filePath: string]: FixChangesSummary[];
  183   } = {};
  184   const { dir, base } = pathLib.parse(entryFileName);
  185   const provenance = await extractProvenance(workspace, dir, dir, base);
  186   const upgradeChanges: FixChangesSummary[] = [];
  187   /* Apply all upgrades first across all files that are included */
  188   for (const fileName of Object.keys(provenance)) {
  189     const skipApplyingPins = true;
  190     const { changes } = await fixIndividualRequirementsTxt(
  191       workspace,
  192       dir,
  193       base,
  194       fileName,
  195       remediation,
  196       provenance[fileName],
  197       options,
  198       skipApplyingPins,
  199     );
  200     upgradeChanges.push(...changes);
  201     fixedMeta[pathLib.normalize(pathLib.join(dir, fileName))] = upgradeChanges;
  202   }
  203 
  204   /* Apply all left over remediation as pins in the entry targetFile */
  205   const toPin: RemediationChanges = filterOutAppliedUpgrades(
  206     remediation,
  207     upgradeChanges,
  208   );
  209   const directUpgradesOnly = false;
  210   const fileForPinning = await selectFileForPinning(entity);
  211   const { changes: pinnedChanges } = await fixIndividualRequirementsTxt(
  212     workspace,
  213     dir,
  214     base,
  215     fileForPinning.fileName,
  216     toPin,
  217     parseRequirementsFile(fileForPinning.fileContent),
  218     options,
  219     directUpgradesOnly,
  220   );
  221 
  222   return { changes: [...upgradeChanges, ...pinnedChanges], fixedMeta };
  223 }
  224 
  225 function filterOutAppliedUpgrades(
  226   remediation: RemediationChanges,
  227   upgradeChanges: FixChangesSummary[],
  228 ): RemediationChanges {
  229   const pinRemediation: RemediationChanges = {
  230     ...remediation,
  231     pin: {}, // delete the pin remediation so we can collect un-applied remediation
  232   };
  233   const pins = remediation.pin;
  234   const normalizedAppliedRemediation = upgradeChanges
  235     .map((c) => {
  236       if (c.success && c.from) {
  237         const [pkgName, versionAndMore] = c.from?.split('@');
  238         return `${standardizePackageName(pkgName)}@${versionAndMore}`;
  239       }
  240       return false;
  241     })
  242     .filter(Boolean);
  243   for (const pkgAtVersion of Object.keys(pins)) {
  244     const [pkgName, versionAndMore] = pkgAtVersion.split('@');
  245     if (
  246       !normalizedAppliedRemediation.includes(
  247         `${standardizePackageName(pkgName)}@${versionAndMore}`,
  248       )
  249     ) {
  250       pinRemediation.pin[pkgAtVersion] = pins[pkgAtVersion];
  251     }
  252   }
  253   return pinRemediation;
  254 }
  255 
  256 function sortByDirectory(
  257   entities: EntityToFix[],
  258 ): {
  259   [dir: string]: Array<{
  260     entity: EntityToFix;
  261     dir: string;
  262     base: string;
  263     ext: string;
  264     root: string;
  265     name: string;
  266   }>;
  267 } {
  268   const mapped = entities.map((e) => ({
  269     entity: e,
  270     ...pathLib.parse(e.scanResult.identity.targetFile!),
  271   }));
  272 
  273   const sorted = sortBy(mapped, 'dir');
  274   return groupBy(sorted, 'dir');
  275 }
  276 
  277 export async function selectFileForPinning(
  278   entity: EntityToFix,
  279 ): Promise<{
  280   fileName: string;
  281   fileContent: string;
  282 }> {
  283   const targetFile = entity.scanResult.identity.targetFile!;
  284   const { dir, base } = pathLib.parse(targetFile);
  285   const { workspace } = entity;
  286   // default to adding pins in the scanned file
  287   let fileName = base;
  288   let requirementsTxt = await workspace.readFile(targetFile);
  289 
  290   const { containsRequire, matches } = await containsRequireDirective(
  291     requirementsTxt,
  292   );
  293   const constraintsMatch = matches.filter((m) => m.includes('c'));
  294   if (containsRequire && constraintsMatch[0]) {
  295     // prefer to pin in constraints file if present
  296     fileName = constraintsMatch[0][2];
  297     requirementsTxt = await workspace.readFile(pathLib.join(dir, fileName));
  298   }
  299   return { fileContent: requirementsTxt, fileName };
  300 }
  301 
  302 function getFixedEntityIssues(
  303   fixedIssueIds: string[],
  304   issues: Issue[],
  305 ): string[] {
  306   const fixed: string[] = [];
  307   for (const { issueId } of issues) {
  308     if (fixedIssueIds.includes(issueId)) {
  309       fixed.push(issueId);
  310     }
  311   }
  312   return fixed;
  313 }