"Fossies" - the Fresh Open Source Software Archive

Member "cli-1.1280.1/packages/cli-alert/src/index.ts" (20 Feb 2024, 7123 Bytes) of package /linux/misc/snyk-cli-1.1280.1.tar.gz:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) TypeScript source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file. See also the latest Fossies "Diffs" side-by-side code changes report for "index.ts": 1.1280.0_vs_1.1280.1.

    1 import { Octokit } from '@octokit/rest';
    2 import { IncomingWebhook } from '@slack/webhook';
    3 import { IncomingWebhookDefaultArguments } from '@slack/webhook';
    4 import { event } from '@pagerduty/pdjs';
    5 
    6 if (
    7   !process.env.USER_GITHUB_TOKEN ||
    8   !process.env.SLACK_WEBHOOK_URL ||
    9   !process.env.PD_ROUTING_KEY
   10 ) {
   11   console.error(
   12     'Missing USER_GITHUB_TOKEN, SLACK_WEBHOOK_URL or PD_ROUTING_KEY',
   13   );
   14   process.exit(1);
   15 }
   16 
   17 const GITHUB_TOKEN = process.env.USER_GITHUB_TOKEN;
   18 const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL;
   19 const PD_ROUTING_KEY = process.env.PD_ROUTING_KEY;
   20 
   21 const octokit = new Octokit({
   22   auth: GITHUB_TOKEN,
   23 });
   24 const slackWebhook = new IncomingWebhook(SLACK_WEBHOOK_URL);
   25 
   26 async function discoverConsecutiveFailures(
   27   firstWorkflowRun: number,
   28   secondWorkflowRun: number,
   29 ): Promise<string[]> {
   30   // Get all jobs of the latest 2 smoke tests
   31   const firstWorkflowRunJobs = (
   32     await octokit.actions.listJobsForWorkflowRun({
   33       owner: 'snyk',
   34       repo: 'snyk',
   35       run_id: firstWorkflowRun,
   36     })
   37   ).data.jobs;
   38 
   39   const secondWorkflowRunJobs = (
   40     await octokit.actions.listJobsForWorkflowRun({
   41       owner: 'snyk',
   42       repo: 'snyk',
   43       run_id: secondWorkflowRun,
   44     })
   45   ).data.jobs;
   46 
   47   const failedJobs: string[] = [];
   48 
   49   // If the same job failed in both smoke tests, it has been failing for 2 hours now. Save job to re-run later.
   50   for (const jobName of firstWorkflowRunJobs.map((j) => j.name)) {
   51     const firstJob = firstWorkflowRunJobs.find((j) => j.name === jobName);
   52     const secondJob = secondWorkflowRunJobs.find((j) => j.name === jobName);
   53 
   54     if (firstJob === undefined || secondJob === undefined) {
   55       console.error(
   56         `Could not find job ${jobName} in Smoke Tests ID: ${
   57           firstJob ? secondWorkflowRun : firstWorkflowRun
   58         }`,
   59       );
   60       process.exit(1);
   61     } else if (
   62       'failure' === firstJob.conclusion &&
   63       firstJob.conclusion === secondJob.conclusion
   64     ) {
   65       console.log(`Found a job that failed 2 times in a row: ${jobName}`);
   66       failedJobs.push(jobName);
   67     }
   68   }
   69 
   70   return failedJobs;
   71 }
   72 
   73 async function sendPagerDuty() {
   74   try {
   75     const res = await event({
   76       data: {
   77         routing_key: PD_ROUTING_KEY,
   78         event_action: 'trigger',
   79         payload: {
   80           summary: 'CLI Alert. Smoke tests failing',
   81           source: 'Snyk CLI Smoke tests',
   82           severity: 'warning',
   83         },
   84         dedup_key: 'b0209ed890d34eb787b3ed58f31553cc',
   85       },
   86     });
   87     console.log(res);
   88   } catch (err) {
   89     console.error(err);
   90     process.exit(1);
   91   }
   92 }
   93 
   94 async function sendSlackAlert(failedJobs: string[]) {
   95   console.log('Jobs failed again. Sending Slack alert...');
   96   const args: IncomingWebhookDefaultArguments = {
   97     username: 'Hammer Alerts',
   98     text: `A Smoke Tests Job failed more than 2 times in a row. \n <https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22|Smoke Tests Results>. \n Failed Job/s: ${failedJobs.join(
   99       ', ',
  100     )}`,
  101     icon_emoji: 'hammer',
  102   };
  103   await slackWebhook.send(args);
  104   console.log('Slack alert sent.');
  105 }
  106 
  107 async function waitForConclusion(runID: number) {
  108   let status: string | null = 'queued';
  109   const before = Date.now();
  110   console.log('Waiting for Smoke Test to finish running...');
  111 
  112   // Wait for run to finish
  113   while (status !== 'completed') {
  114     const smokeTest = (
  115       await octokit.actions.getWorkflowRun({
  116         owner: 'snyk',
  117         repo: 'snyk',
  118         run_id: runID,
  119       })
  120     ).data;
  121 
  122     // Wait 30 seconds
  123     await new Promise((r) => setTimeout(r, 30_000));
  124     status = smokeTest.status;
  125     const time = (Date.now() - before) / 1000;
  126     const minutes = Math.floor(time / 60);
  127     console.log(
  128       `Current smoke test status: "${status}". Elapsed: ${minutes} minute${
  129         minutes !== 1 ? 's' : ''
  130       }`,
  131     );
  132   }
  133   console.log('Finished run.');
  134 }
  135 
  136 async function checkJobConclusion(
  137   runID: number,
  138   failedJobs: string[],
  139 ): Promise<string[]> {
  140   // Get conclusions of the jobs that failed before
  141   const workflowRunJobs = (
  142     await octokit.actions.listJobsForWorkflowRun({
  143       owner: 'snyk',
  144       repo: 'snyk',
  145       run_id: runID,
  146     })
  147   ).data.jobs;
  148 
  149   // Return false if jobs that failed before failed again
  150   const rerunJobs = workflowRunJobs.filter((job) =>
  151     failedJobs.includes(job.name),
  152   );
  153 
  154   const failedAgainJobs: string[] = [];
  155   for (const job of rerunJobs) {
  156     if (job.conclusion === 'failure') {
  157       failedAgainJobs.push(job.name);
  158     }
  159   }
  160 
  161   return failedAgainJobs;
  162 }
  163 
  164 async function run() {
  165   try {
  166     // Get ID of smoke tests workflow
  167     const allWorkflows = (
  168       await octokit.actions.listRepoWorkflows({
  169         owner: 'snyk',
  170         repo: 'snyk',
  171       })
  172     ).data;
  173 
  174     const smokeTestsID = allWorkflows.workflows.find(
  175       (workflow) => workflow.name === 'Smoke Tests',
  176     )?.id;
  177 
  178     if (!smokeTestsID) {
  179       console.error('Error: Could not find Smoke Tests workflow ID');
  180       process.exit(1);
  181     }
  182 
  183     // Get latest smoke tests
  184     const workflowRuns = (
  185       await octokit.actions.listWorkflowRuns({
  186         owner: 'snyk',
  187         repo: 'snyk',
  188         branch: 'main',
  189         workflow_id: smokeTestsID,
  190       })
  191     ).data;
  192     console.log('Got latest smoke tests...');
  193 
  194     // Check the latest 2 smoke tests for tests that had the same job fail 2 times in a row.
  195     const latestWorkflowRuns = workflowRuns.workflow_runs.slice(0, 2);
  196     const id = latestWorkflowRuns[0].id;
  197 
  198     // Check current status of smoke test workflow and wait if it's still running
  199     const latestRun = (
  200       await octokit.actions.getWorkflowRun({
  201         owner: 'snyk',
  202         repo: 'snyk',
  203         run_id: id,
  204       })
  205     ).data;
  206 
  207     if (latestRun.status !== 'completed') {
  208       console.log('First wait for current run to finish...');
  209       await waitForConclusion(id);
  210     }
  211 
  212     console.log('Checking smoke tests jobs...');
  213     const failedWorkflows = await discoverConsecutiveFailures(
  214       latestWorkflowRuns[0].id,
  215       latestWorkflowRuns[1].id,
  216     );
  217 
  218     if (!failedWorkflows.length) {
  219       console.log(
  220         'There were no 2 consecutive fails on a job. No need to alert.',
  221       );
  222       return;
  223     }
  224 
  225     console.log('Trying to re-run smoke test...');
  226 
  227     // After making sure smoke test isn't currently running - try to re-run
  228     console.log(`Starting re-run of Smoke Test. ID number: ${id}...`);
  229     await octokit.actions.reRunWorkflow({
  230       owner: 'snyk',
  231       repo: 'snyk',
  232       run_id: id,
  233     });
  234 
  235     // Wait for run to finish
  236     await waitForConclusion(id);
  237     const failedAgainJobs = await checkJobConclusion(id, failedWorkflows);
  238     console.log('Re-run completed.');
  239 
  240     // If run failed again, send Slack alert and PagerDuty
  241     if (failedAgainJobs.length > 0) {
  242       await sendSlackAlert(failedAgainJobs);
  243       await sendPagerDuty();
  244     } else {
  245       console.log('Jobs succeeded after re-run. Do not alert.');
  246     }
  247   } catch (error) {
  248     console.error(error);
  249     process.exit(1);
  250   }
  251 }
  252 
  253 process.on('uncaughtException', (err) => {
  254   console.error(err);
  255   process.exit(1);
  256 });
  257 
  258 // Exec
  259 run();