"Fossies" - the Fresh Open Source Software Archive

Member "Atom/resources/app/apm/node_modules/npm/lib/cache/add-remote-git.js" (11 Apr 2017, 15945 Bytes) of package /windows/misc/atom-windows.zip:


As a special service "Fossies" has tried to format the requested source page into HTML format using (guessed) Javascript source code syntax highlighting (style: standard) with prefixed line numbers and code folding option. Alternatively you can here view or download the uninterpreted source code file.

    1 var assert = require('assert')
    2 var fs = require('graceful-fs')
    3 var path = require('path')
    4 var url = require('url')
    5 
    6 var chownr = require('chownr')
    7 var dezalgo = require('dezalgo')
    8 var hostedFromURL = require('hosted-git-info').fromUrl
    9 var inflight = require('inflight')
   10 var log = require('npmlog')
   11 var mkdir = require('mkdirp')
   12 var normalizeGitUrl = require('normalize-git-url')
   13 var npa = require('npm-package-arg')
   14 var realizePackageSpecifier = require('realize-package-specifier')
   15 var uniqueFilename = require('unique-filename')
   16 
   17 var addLocal = require('./add-local.js')
   18 var correctMkdir = require('../utils/correct-mkdir.js')
   19 var git = require('../utils/git.js')
   20 var npm = require('../npm.js')
   21 var rm = require('../utils/gently-rm.js')
   22 var tempFilename = require('../utils/temp-filename.js')
   23 
   24 var remotes = path.resolve(npm.config.get('cache'), '_git-remotes')
   25 var templates = path.join(remotes, '_templates')
   26 
   27 var VALID_VARIABLES = [
   28   'GIT_ASKPASS',
   29   'GIT_PROXY_COMMAND',
   30   'GIT_SSH',
   31   'GIT_SSH_COMMAND',
   32   'GIT_SSL_CAINFO',
   33   'GIT_SSL_NO_VERIFY'
   34 ]
   35 
   36 module.exports = addRemoteGit
   37 function addRemoteGit (uri, _cb) {
   38   assert(typeof uri === 'string', 'must have git URL')
   39   assert(typeof _cb === 'function', 'must have callback')
   40   var cb = dezalgo(_cb)
   41 
   42   log.verbose('addRemoteGit', 'caching', uri)
   43 
   44   // the URL comes in exactly as it was passed on the command line, or as
   45   // normalized by normalize-package-data / read-package-json / read-installed,
   46   // so figure out what to do with it using hosted-git-info
   47   var parsed = hostedFromURL(uri)
   48   if (parsed) {
   49     // normalize GitHub syntax to org/repo (for now)
   50     var from
   51     if (parsed.type === 'github' && parsed.getDefaultRepresentation() === 'shortcut') {
   52       from = parsed.path()
   53     } else {
   54       from = parsed.toString()
   55     }
   56 
   57     log.verbose('addRemoteGit', from, 'is a repository hosted by', parsed.type)
   58 
   59     // prefer explicit URLs to pushing everything through shortcuts
   60     if (parsed.getDefaultRepresentation() !== 'shortcut') {
   61       return tryClone(from, parsed.toString(), false, cb)
   62     }
   63 
   64     // try git:, then git+ssh:, then git+https: before failing
   65     tryGitProto(from, parsed, cb)
   66   } else {
   67     // verify that this is a Git URL before continuing
   68     parsed = npa(uri)
   69     if (parsed.type !== 'git') {
   70       return cb(new Error(uri + 'is not a Git or GitHub URL'))
   71     }
   72 
   73     tryClone(parsed.rawSpec, uri, false, cb)
   74   }
   75 }
   76 
   77 function tryGitProto (from, hostedInfo, cb) {
   78   var gitURL = hostedInfo.git()
   79   if (!gitURL) return tryHTTPS(from, hostedInfo, cb)
   80 
   81   log.silly('tryGitProto', 'attempting to clone', gitURL)
   82   tryClone(from, gitURL, true, function (er) {
   83     if (er) return tryHTTPS(from, hostedInfo, cb)
   84 
   85     cb.apply(this, arguments)
   86   })
   87 }
   88 
   89 function tryHTTPS (from, hostedInfo, cb) {
   90   var httpsURL = hostedInfo.https()
   91   if (!httpsURL) {
   92     return cb(new Error(from + ' can not be cloned via Git, SSH, or HTTPS'))
   93   }
   94 
   95   log.silly('tryHTTPS', 'attempting to clone', httpsURL)
   96   tryClone(from, httpsURL, true, function (er) {
   97     if (er) return trySSH(from, hostedInfo, cb)
   98 
   99     cb.apply(this, arguments)
  100   })
  101 }
  102 
  103 function trySSH (from, hostedInfo, cb) {
  104   var sshURL = hostedInfo.ssh()
  105   if (!sshURL) return tryHTTPS(from, hostedInfo, cb)
  106 
  107   log.silly('trySSH', 'attempting to clone', sshURL)
  108   tryClone(from, sshURL, false, cb)
  109 }
  110 
  111 function tryClone (from, combinedURL, silent, cb) {
  112   log.silly('tryClone', 'cloning', from, 'via', combinedURL)
  113 
  114   var normalized = normalizeGitUrl(combinedURL)
  115   var cloneURL = normalized.url
  116   var treeish = normalized.branch
  117 
  118   // ensure that similarly-named remotes don't collide
  119   var cachedRemote = uniqueFilename(remotes, combinedURL.replace(/[^a-zA-Z0-9]+/g, '-'), cloneURL)
  120   var repoID = path.relative(remotes, cachedRemote)
  121   cachedRemote = path.join(remotes, repoID)
  122 
  123   cb = inflight(repoID, cb)
  124   if (!cb) {
  125     return log.verbose('tryClone', repoID, 'already in flight; waiting')
  126   }
  127   log.verbose('tryClone', repoID, 'not in flight; caching')
  128 
  129   // initialize the remotes cache with the correct perms
  130   getGitDir(function (er) {
  131     if (er) return cb(er)
  132     fs.stat(cachedRemote, function (er, s) {
  133       if (er) return mirrorRemote(from, cloneURL, treeish, cachedRemote, silent, finish)
  134       if (!s.isDirectory()) return resetRemote(from, cloneURL, treeish, cachedRemote, finish)
  135 
  136       validateExistingRemote(from, cloneURL, treeish, cachedRemote, finish)
  137     })
  138 
  139     // always set permissions on the cached remote
  140     function finish (er, data) {
  141       if (er) return cb(er, data)
  142       addModeRecursive(cachedRemote, npm.modes.file, function (er) {
  143         return cb(er, data)
  144       })
  145     }
  146   })
  147 }
  148 
  149 // don't try too hard to hold on to a remote
  150 function resetRemote (from, cloneURL, treeish, cachedRemote, cb) {
  151   log.info('resetRemote', 'resetting', cachedRemote, 'for', from)
  152   rm(cachedRemote, function (er) {
  153     if (er) return cb(er)
  154     mirrorRemote(from, cloneURL, treeish, cachedRemote, false, cb)
  155   })
  156 }
  157 
  158 // reuse a cached remote when possible, but nuke it if it's in an
  159 // inconsistent state
  160 function validateExistingRemote (from, cloneURL, treeish, cachedRemote, cb) {
  161   git.whichAndExec(
  162     ['config', '--get', 'remote.origin.url'],
  163     { cwd: cachedRemote, env: gitEnv() },
  164     function (er, stdout, stderr) {
  165       var originURL
  166       if (stdout) {
  167         originURL = stdout.trim()
  168         log.silly('validateExistingRemote', from, 'remote.origin.url:', originURL)
  169       }
  170 
  171       if (stderr) stderr = stderr.trim()
  172       if (stderr || er) {
  173         log.warn('addRemoteGit', from, 'resetting remote', cachedRemote, 'because of error:', stderr || er)
  174         return resetRemote(from, cloneURL, treeish, cachedRemote, cb)
  175       } else if (cloneURL !== originURL) {
  176         log.warn(
  177           'addRemoteGit',
  178           from,
  179           'pre-existing cached repo', cachedRemote, 'points to', originURL, 'and not', cloneURL
  180         )
  181         return resetRemote(from, cloneURL, treeish, cachedRemote, cb)
  182       }
  183 
  184       log.verbose('validateExistingRemote', from, 'is updating existing cached remote', cachedRemote)
  185       updateRemote(from, cloneURL, treeish, cachedRemote, cb)
  186     }
  187   )
  188 }
  189 
  190 // make a complete bare mirror of the remote repo
  191 // NOTE: npm uses a blank template directory to prevent weird inconsistencies
  192 // https://github.com/npm/npm/issues/5867
  193 function mirrorRemote (from, cloneURL, treeish, cachedRemote, silent, cb) {
  194   mkdir(cachedRemote, function (er) {
  195     if (er) return cb(er)
  196 
  197     var args = [
  198       'clone',
  199       '--template=' + templates,
  200       '--mirror',
  201       cloneURL, cachedRemote
  202     ]
  203     git.whichAndExec(
  204       ['clone', '--template=' + templates, '--mirror', cloneURL, cachedRemote],
  205       { cwd: cachedRemote, env: gitEnv() },
  206       function (er, stdout, stderr) {
  207         if (er) {
  208           var combined = (stdout + '\n' + stderr).trim()
  209           var command = 'git ' + args.join(' ') + ':'
  210           if (silent) {
  211             log.verbose(command, combined)
  212           } else {
  213             log.error(command, combined)
  214           }
  215           return cb(er)
  216         }
  217         log.verbose('mirrorRemote', from, 'git clone ' + cloneURL, stdout.trim())
  218         setPermissions(from, cloneURL, treeish, cachedRemote, cb)
  219       }
  220     )
  221   })
  222 }
  223 
  224 function setPermissions (from, cloneURL, treeish, cachedRemote, cb) {
  225   if (process.platform === 'win32') {
  226     log.verbose('setPermissions', from, 'skipping chownr on Windows')
  227     resolveHead(from, cloneURL, treeish, cachedRemote, cb)
  228   } else {
  229     getGitDir(function (er, cs) {
  230       if (er) {
  231         log.error('setPermissions', from, 'could not get cache stat')
  232         return cb(er)
  233       }
  234 
  235       chownr(cachedRemote, cs.uid, cs.gid, function (er) {
  236         if (er) {
  237           log.error(
  238             'setPermissions',
  239             'Failed to change git repository ownership under npm cache for',
  240             cachedRemote
  241           )
  242           return cb(er)
  243         }
  244 
  245         log.verbose('setPermissions', from, 'set permissions on', cachedRemote)
  246         resolveHead(from, cloneURL, treeish, cachedRemote, cb)
  247       })
  248     })
  249   }
  250 }
  251 
  252 // always fetch the origin, even right after mirroring, because this way
  253 // permissions will get set correctly
  254 function updateRemote (from, cloneURL, treeish, cachedRemote, cb) {
  255   git.whichAndExec(
  256     ['fetch', '-a', 'origin'],
  257     { cwd: cachedRemote, env: gitEnv() },
  258     function (er, stdout, stderr) {
  259       if (er) {
  260         var combined = (stdout + '\n' + stderr).trim()
  261         log.error('git fetch -a origin (' + cloneURL + ')', combined)
  262         return cb(er)
  263       }
  264       log.verbose('updateRemote', 'git fetch -a origin (' + cloneURL + ')', stdout.trim())
  265 
  266       setPermissions(from, cloneURL, treeish, cachedRemote, cb)
  267     }
  268   )
  269 }
  270 
  271 // branches and tags are both symbolic labels that can be attached to different
  272 // commits, so resolve the commit-ish to the current actual treeish the label
  273 // corresponds to
  274 //
  275 // important for shrinkwrap
  276 function resolveHead (from, cloneURL, treeish, cachedRemote, cb) {
  277   log.verbose('resolveHead', from, 'original treeish:', treeish)
  278   var args = ['rev-list', '-n1', treeish]
  279   git.whichAndExec(
  280     args,
  281     { cwd: cachedRemote, env: gitEnv() },
  282     function (er, stdout, stderr) {
  283       if (er) {
  284         log.error('git ' + args.join(' ') + ':', stderr)
  285         return cb(er)
  286       }
  287 
  288       var resolvedTreeish = stdout.trim()
  289       log.silly('resolveHead', from, 'resolved treeish:', resolvedTreeish)
  290 
  291       var resolvedURL = getResolved(cloneURL, resolvedTreeish)
  292       if (!resolvedURL) {
  293         return cb(new Error(
  294           'unable to clone ' + from + ' because git clone string ' +
  295             cloneURL + ' is in a form npm can\'t handle'
  296         ))
  297       }
  298       log.verbose('resolveHead', from, 'resolved Git URL:', resolvedURL)
  299 
  300       // generate a unique filename
  301       var tmpdir = path.join(tempFilename('git-cache'), resolvedTreeish)
  302       log.silly('resolveHead', 'Git working directory:', tmpdir)
  303 
  304       mkdir(tmpdir, function (er) {
  305         if (er) return cb(er)
  306 
  307         cloneResolved(from, resolvedURL, resolvedTreeish, cachedRemote, tmpdir, cb)
  308       })
  309     }
  310   )
  311 }
  312 
  313 // make a clone from the mirrored cache so we have a temporary directory in
  314 // which we can check out the resolved treeish
  315 function cloneResolved (from, resolvedURL, resolvedTreeish, cachedRemote, tmpdir, cb) {
  316   var args = ['clone', cachedRemote, tmpdir]
  317   git.whichAndExec(
  318     args,
  319     { cwd: cachedRemote, env: gitEnv() },
  320     function (er, stdout, stderr) {
  321       stdout = (stdout + '\n' + stderr).trim()
  322       if (er) {
  323         log.error('git ' + args.join(' ') + ':', stderr)
  324         return cb(er)
  325       }
  326       log.verbose('cloneResolved', from, 'clone', stdout)
  327 
  328       checkoutTreeish(from, resolvedURL, resolvedTreeish, tmpdir, cb)
  329     }
  330   )
  331 }
  332 
  333 // there is no safe way to do a one-step clone to a treeish that isn't
  334 // guaranteed to be a branch, so explicitly check out the treeish once it's
  335 // cloned
  336 function checkoutTreeish (from, resolvedURL, resolvedTreeish, tmpdir, cb) {
  337   var args = ['checkout', resolvedTreeish]
  338   git.whichAndExec(
  339     args,
  340     { cwd: tmpdir, env: gitEnv() },
  341     function (er, stdout, stderr) {
  342       stdout = (stdout + '\n' + stderr).trim()
  343       if (er) {
  344         log.error('git ' + args.join(' ') + ':', stderr)
  345         return cb(er)
  346       }
  347       log.verbose('checkoutTreeish', from, 'checkout', stdout)
  348 
  349       updateSubmodules(from, resolvedURL, tmpdir, cb)
  350     }
  351   )
  352 }
  353 
  354 function updateSubmodules (from, resolvedURL, tmpdir, cb) {
  355   var args = ['submodule', '-q', 'update', '--init', '--recursive']
  356   git.whichAndExec(
  357     args,
  358     { cwd: tmpdir, env: gitEnv() },
  359     function (er, stdout, stderr) {
  360       stdout = (stdout + '\n' + stderr).trim()
  361       if (er) {
  362         log.error('git ' + args.join(' ') + ':', stderr)
  363         return cb(er)
  364       }
  365       log.verbose('updateSubmodules', from, 'submodule update', stdout)
  366 
  367       // convince addLocal that the checkout is a local dependency
  368       realizePackageSpecifier(tmpdir, function (er, spec) {
  369         if (er) {
  370           log.error('addRemoteGit', 'Failed to map', tmpdir, 'to a package specifier')
  371           return cb(er)
  372         }
  373 
  374         // ensure pack logic is applied
  375         // https://github.com/npm/npm/issues/6400
  376         addLocal(spec, null, function (er, data) {
  377           if (data) {
  378             if (npm.config.get('save-exact')) {
  379               log.verbose('addRemoteGit', 'data._from:', resolvedURL, '(save-exact)')
  380               data._from = resolvedURL
  381             } else {
  382               log.verbose('addRemoteGit', 'data._from:', from)
  383               data._from = from
  384             }
  385 
  386             log.verbose('addRemoteGit', 'data._resolved:', resolvedURL)
  387             data._resolved = resolvedURL
  388           }
  389 
  390           cb(er, data)
  391         })
  392       })
  393     }
  394   )
  395 }
  396 
  397 function getGitDir (cb) {
  398   correctMkdir(remotes, function (er, stats) {
  399     if (er) return cb(er)
  400 
  401     // We don't need global templates when cloning. Use an empty directory for
  402     // the templates, creating it (and setting its permissions) if necessary.
  403     mkdir(templates, function (er) {
  404       if (er) return cb(er)
  405 
  406       // Ensure that both the template and remotes directories have the correct
  407       // permissions.
  408       fs.chown(templates, stats.uid, stats.gid, function (er) {
  409         cb(er, stats)
  410       })
  411     })
  412   })
  413 }
  414 
  415 var gitEnv_
  416 function gitEnv () {
  417   // git responds to env vars in some weird ways in post-receive hooks
  418   // so don't carry those along.
  419   if (gitEnv_) return gitEnv_
  420 
  421   // allow users to override npm's insistence on not prompting for
  422   // passphrases, but default to just failing when credentials
  423   // aren't available
  424   gitEnv_ = { GIT_ASKPASS: 'echo' }
  425 
  426   for (var k in process.env) {
  427     if (!~VALID_VARIABLES.indexOf(k) && k.match(/^GIT/)) continue
  428     gitEnv_[k] = process.env[k]
  429   }
  430   return gitEnv_
  431 }
  432 
  433 addRemoteGit.getResolved = getResolved
  434 function getResolved (uri, treeish) {
  435   // normalize hosted-git-info clone URLs back into regular URLs
  436   // this will only work on URLs that hosted-git-info recognizes
  437   // https://github.com/npm/npm/issues/7961
  438   var rehydrated = hostedFromURL(uri)
  439   if (rehydrated) uri = rehydrated.toString()
  440 
  441   var parsed = url.parse(uri)
  442 
  443   // Checks for known protocols:
  444   // http:, https:, ssh:, and git:, with optional git+ prefix.
  445   if (!parsed.protocol ||
  446       !parsed.protocol.match(/^(((git\+)?(https?|ssh|file))|git|file):$/)) {
  447     uri = 'git+ssh://' + uri
  448   }
  449 
  450   if (!/^git[+:]/.test(uri)) {
  451     uri = 'git+' + uri
  452   }
  453 
  454   // Not all URIs are actually URIs, so use regex for the treeish.
  455   return uri.replace(/(?:#.*)?$/, '#' + treeish)
  456 }
  457 
  458 // similar to chmodr except it add permissions rather than overwriting them
  459 // adapted from https://github.com/isaacs/chmodr/blob/master/chmodr.js
  460 function addModeRecursive (cachedRemote, mode, cb) {
  461   fs.readdir(cachedRemote, function (er, children) {
  462     // Any error other than ENOTDIR means it's not readable, or doesn't exist.
  463     // Give up.
  464     if (er && er.code !== 'ENOTDIR') return cb(er)
  465     if (er || !children.length) return addMode(cachedRemote, mode, cb)
  466 
  467     var len = children.length
  468     var errState = null
  469     children.forEach(function (child) {
  470       addModeRecursive(path.resolve(cachedRemote, child), mode, then)
  471     })
  472 
  473     function then (er) {
  474       if (errState) return undefined
  475       if (er) return cb(errState = er)
  476       if (--len === 0) return addMode(cachedRemote, dirMode(mode), cb)
  477     }
  478   })
  479 }
  480 
  481 function addMode (cachedRemote, mode, cb) {
  482   fs.stat(cachedRemote, function (er, stats) {
  483     if (er) return cb(er)
  484     mode = stats.mode | mode
  485     fs.chmod(cachedRemote, mode, cb)
  486   })
  487 }
  488 
  489 // taken from https://github.com/isaacs/chmodr/blob/master/chmodr.js
  490 function dirMode (mode) {
  491   if (mode & parseInt('0400', 8)) mode |= parseInt('0100', 8)
  492   if (mode & parseInt('040', 8)) mode |= parseInt('010', 8)
  493   if (mode & parseInt('04', 8)) mode |= parseInt('01', 8)
  494   return mode
  495 }