This commit is contained in:
2022-07-18 02:50:52 +00:00
parent befd344ab0
commit 06181b34d6
8569 changed files with 818704 additions and 352705 deletions
+4 -3
View File
@@ -15,9 +15,10 @@ module.exports = contentPath
function contentPath (cache, integrity) {
const sri = ssri.parse(integrity, { single: true })
// contentPath is the *strongest* algo given
return path.join.apply(
path,
[contentDir(cache), sri.algorithm].concat(hashToSegments(sri.hexDigest()))
return path.join(
contentDir(cache),
sri.algorithm,
...hashToSegments(sri.hexDigest())
)
}
+45 -62
View File
@@ -2,8 +2,7 @@
const util = require('util')
const figgyPudding = require('figgy-pudding')
const fs = require('graceful-fs')
const fs = require('fs')
const fsm = require('fs-minipass')
const ssri = require('ssri')
const contentPath = require('./path')
@@ -12,30 +11,25 @@ const Pipeline = require('minipass-pipeline')
const lstat = util.promisify(fs.lstat)
const readFile = util.promisify(fs.readFile)
const ReadOpts = figgyPudding({
size: {}
})
module.exports = read
const MAX_SINGLE_READ_SIZE = 64 * 1024 * 1024
function read (cache, integrity, opts) {
opts = ReadOpts(opts)
function read (cache, integrity, opts = {}) {
const { size } = opts
return withContentSri(cache, integrity, (cpath, sri) => {
// get size
return lstat(cpath).then(stat => ({ stat, cpath, sri }))
}).then(({ stat, cpath, sri }) => {
if (typeof opts.size === 'number' && stat.size !== opts.size) {
throw sizeError(opts.size, stat.size)
}
if (stat.size > MAX_SINGLE_READ_SIZE) {
if (typeof size === 'number' && stat.size !== size)
throw sizeError(size, stat.size)
if (stat.size > MAX_SINGLE_READ_SIZE)
return readPipeline(cpath, stat.size, sri, new Pipeline()).concat()
}
return readFile(cpath, null).then((data) => {
if (!ssri.checkData(data, sri)) {
if (!ssri.checkData(data, sri))
throw integrityError(sri, cpath)
}
return data
})
})
@@ -45,11 +39,11 @@ const readPipeline = (cpath, size, sri, stream) => {
stream.push(
new fsm.ReadStream(cpath, {
size,
readSize: MAX_SINGLE_READ_SIZE
readSize: MAX_SINGLE_READ_SIZE,
}),
ssri.integrityStream({
integrity: sri,
size
size,
})
)
return stream
@@ -57,17 +51,15 @@ const readPipeline = (cpath, size, sri, stream) => {
module.exports.sync = readSync
function readSync (cache, integrity, opts) {
opts = ReadOpts(opts)
function readSync (cache, integrity, opts = {}) {
const { size } = opts
return withContentSriSync(cache, integrity, (cpath, sri) => {
const data = fs.readFileSync(cpath)
if (typeof opts.size === 'number' && opts.size !== data.length) {
throw sizeError(opts.size, data.length)
}
if (typeof size === 'number' && size !== data.length)
throw sizeError(size, data.length)
if (ssri.checkData(data, sri)) {
if (ssri.checkData(data, sri))
return data
}
throw integrityError(sri, cpath)
})
@@ -76,17 +68,16 @@ function readSync (cache, integrity, opts) {
module.exports.stream = readStream
module.exports.readStream = readStream
function readStream (cache, integrity, opts) {
opts = ReadOpts(opts)
function readStream (cache, integrity, opts = {}) {
const { size } = opts
const stream = new Pipeline()
withContentSri(cache, integrity, (cpath, sri) => {
// just lstat to ensure it exists
return lstat(cpath).then((stat) => ({ stat, cpath, sri }))
}).then(({ stat, cpath, sri }) => {
if (typeof opts.size === 'number' && opts.size !== stat.size) {
return stream.emit('error', sizeError(opts.size, stat.size))
}
if (typeof size === 'number' && size !== stat.size)
return stream.emit('error', sizeError(size, stat.size))
readPipeline(cpath, stat.size, sri, stream)
}, er => stream.emit('error', er))
@@ -100,15 +91,13 @@ if (fs.copyFile) {
copyFile = util.promisify(fs.copyFile)
}
function copy (cache, integrity, dest, opts) {
opts = ReadOpts(opts)
function copy (cache, integrity, dest) {
return withContentSri(cache, integrity, (cpath, sri) => {
return copyFile(cpath, dest)
})
}
function copySync (cache, integrity, dest, opts) {
opts = ReadOpts(opts)
function copySync (cache, integrity, dest) {
return withContentSriSync(cache, integrity, (cpath, sri) => {
return fs.copyFileSync(cpath, dest)
})
@@ -117,21 +106,21 @@ function copySync (cache, integrity, dest, opts) {
module.exports.hasContent = hasContent
function hasContent (cache, integrity) {
if (!integrity) {
if (!integrity)
return Promise.resolve(false)
}
return withContentSri(cache, integrity, (cpath, sri) => {
return lstat(cpath).then((stat) => ({ size: stat.size, sri, stat }))
}).catch((err) => {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return false
}
if (err.code === 'EPERM') {
if (process.platform !== 'win32') {
/* istanbul ignore else */
if (process.platform !== 'win32')
throw err
} else {
else
return false
}
}
})
}
@@ -139,23 +128,23 @@ function hasContent (cache, integrity) {
module.exports.hasContent.sync = hasContentSync
function hasContentSync (cache, integrity) {
if (!integrity) {
if (!integrity)
return false
}
return withContentSriSync(cache, integrity, (cpath, sri) => {
try {
const stat = fs.lstatSync(cpath)
return { size: stat.size, sri, stat }
} catch (err) {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return false
}
if (err.code === 'EPERM') {
if (process.platform !== 'win32') {
/* istanbul ignore else */
if (process.platform !== 'win32')
throw err
} else {
else
return false
}
}
}
})
@@ -173,9 +162,10 @@ function withContentSri (cache, integrity, fn) {
const cpath = contentPath(cache, digests[0])
return fn(cpath, digests[0])
} else {
// Can't use race here because a generic error can happen before a ENOENT error, and can happen before a valid result
// Can't use race here because a generic error can happen before
// a ENOENT error, and can happen before a valid result
return Promise
.all(sri[sri.pickAlgorithm()].map((meta) => {
.all(digests.map((meta) => {
return withContentSri(cache, meta, fn)
.catch((err) => {
if (err.code === 'ENOENT') {
@@ -190,21 +180,16 @@ function withContentSri (cache, integrity, fn) {
.then((results) => {
// Return the first non error if it is found
const result = results.find((r) => !(r instanceof Error))
if (result) {
if (result)
return result
}
// Throw the No matching content found error
const enoentError = results.find((r) => r.code === 'ENOENT')
if (enoentError) {
if (enoentError)
throw enoentError
}
// Throw generic error
const genericError = results.find((r) => r instanceof Error)
if (genericError) {
throw genericError
}
throw results.find((r) => r instanceof Error)
})
}
}
@@ -231,16 +216,14 @@ function withContentSriSync (cache, integrity, fn) {
return fn(cpath, digests[0])
} else {
let lastErr = null
for (const meta of sri[sri.pickAlgorithm()]) {
for (const meta of digests) {
try {
return withContentSriSync(cache, meta, fn)
} catch (err) {
lastErr = err
}
}
if (lastErr) {
throw lastErr
}
throw lastErr
}
}
+4 -7
View File
@@ -10,13 +10,10 @@ module.exports = rm
function rm (cache, integrity) {
return hasContent(cache, integrity).then((content) => {
if (content) {
const sri = content.sri
if (sri) {
return rimraf(contentPath(cache, sri)).then(() => true)
}
} else {
// ~pretty~ sure we can't end up with a content lacking sri, but be safe
if (content && content.sri)
return rimraf(contentPath(cache, content.sri)).then(() => true)
else
return false
}
})
}
+26 -24
View File
@@ -4,7 +4,7 @@ const util = require('util')
const contentPath = require('./path')
const fixOwner = require('../util/fix-owner')
const fs = require('graceful-fs')
const fs = require('fs')
const moveFile = require('../util/move-file')
const Minipass = require('minipass')
const Pipeline = require('minipass-pipeline')
@@ -20,20 +20,17 @@ const writeFile = util.promisify(fs.writeFile)
module.exports = write
function write (cache, data, opts) {
opts = opts || {}
if (opts.algorithms && opts.algorithms.length > 1) {
function write (cache, data, opts = {}) {
const { algorithms, size, integrity } = opts
if (algorithms && algorithms.length > 1)
throw new Error('opts.algorithms only supports a single algorithm for now')
}
if (typeof opts.size === 'number' && data.length !== opts.size) {
return Promise.reject(sizeError(opts.size, data.length))
}
const sri = ssri.fromData(data, {
algorithms: opts.algorithms
})
if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) {
return Promise.reject(checksumError(opts.integrity, sri))
}
if (typeof size === 'number' && data.length !== size)
return Promise.reject(sizeError(size, data.length))
const sri = ssri.fromData(data, algorithms ? { algorithms } : {})
if (integrity && !ssri.checkData(data, integrity, opts))
return Promise.reject(checksumError(integrity, sri))
return disposer(makeTmp(cache, opts), makeTmpDisposer,
(tmp) => {
@@ -90,8 +87,7 @@ class CacacheWriteStream extends Flush {
}
}
function writeStream (cache, opts) {
opts = opts || {}
function writeStream (cache, opts = {}) {
return new CacacheWriteStream(cache, opts)
}
@@ -115,13 +111,17 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
const hashStream = ssri.integrityStream({
integrity: opts.integrity,
algorithms: opts.algorithms,
size: opts.size
size: opts.size,
})
hashStream.on('integrity', i => {
integrity = i
})
hashStream.on('size', s => {
size = s
})
hashStream.on('integrity', i => { integrity = i })
hashStream.on('size', s => { size = s })
const outStream = new fsm.WriteStream(tmpTarget, {
flags: 'wx'
flags: 'wx',
})
// NB: this can throw if the hashStream has a problem with
@@ -135,21 +135,23 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
return pipeline.promise()
.then(() => ({ integrity, size }))
.catch(er => rimraf(tmpTarget).then(() => { throw er }))
.catch(er => rimraf(tmpTarget).then(() => {
throw er
}))
}
function makeTmp (cache, opts) {
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
return fixOwner.mkdirfix(cache, path.dirname(tmpTarget)).then(() => ({
target: tmpTarget,
moved: false
moved: false,
}))
}
function makeTmpDisposer (tmp) {
if (tmp.moved) {
if (tmp.moved)
return Promise.resolve()
}
return rimraf(tmp.target)
}
+133 -50
View File
@@ -1,21 +1,27 @@
'use strict'
const util = require('util')
const crypto = require('crypto')
const figgyPudding = require('figgy-pudding')
const fs = require('graceful-fs')
const fs = require('fs')
const Minipass = require('minipass')
const path = require('path')
const ssri = require('ssri')
const uniqueFilename = require('unique-filename')
const { disposer } = require('./util/disposer')
const contentPath = require('./content/path')
const fixOwner = require('./util/fix-owner')
const hashToSegments = require('./util/hash-to-segments')
const indexV = require('../package.json')['cache-version'].index
const moveFile = require('@npmcli/move-file')
const _rimraf = require('rimraf')
const rimraf = util.promisify(_rimraf)
rimraf.sync = _rimraf.sync
const appendFile = util.promisify(fs.appendFile)
const readFile = util.promisify(fs.readFile)
const readdir = util.promisify(fs.readdir)
const writeFile = util.promisify(fs.writeFile)
module.exports.NotFoundError = class NotFoundError extends Error {
constructor (cache, key) {
@@ -26,22 +32,93 @@ module.exports.NotFoundError = class NotFoundError extends Error {
}
}
const IndexOpts = figgyPudding({
metadata: {},
size: {}
})
module.exports.compact = compact
async function compact (cache, key, matchFn, opts = {}) {
const bucket = bucketPath(cache, key)
const entries = await bucketEntries(bucket)
const newEntries = []
// we loop backwards because the bottom-most result is the newest
// since we add new entries with appendFile
for (let i = entries.length - 1; i >= 0; --i) {
const entry = entries[i]
// a null integrity could mean either a delete was appended
// or the user has simply stored an index that does not map
// to any content. we determine if the user wants to keep the
// null integrity based on the validateEntry function passed in options.
// if the integrity is null and no validateEntry is provided, we break
// as we consider the null integrity to be a deletion of everything
// that came before it.
if (entry.integrity === null && !opts.validateEntry)
break
// if this entry is valid, and it is either the first entry or
// the newEntries array doesn't already include an entry that
// matches this one based on the provided matchFn, then we add
// it to the beginning of our list
if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&
(newEntries.length === 0 ||
!newEntries.find((oldEntry) => matchFn(oldEntry, entry))))
newEntries.unshift(entry)
}
const newIndex = '\n' + newEntries.map((entry) => {
const stringified = JSON.stringify(entry)
const hash = hashEntry(stringified)
return `${hash}\t${stringified}`
}).join('\n')
const setup = async () => {
const target = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
await fixOwner.mkdirfix(cache, path.dirname(target))
return {
target,
moved: false,
}
}
const teardown = async (tmp) => {
if (!tmp.moved)
return rimraf(tmp.target)
}
const write = async (tmp) => {
await writeFile(tmp.target, newIndex, { flag: 'wx' })
await fixOwner.mkdirfix(cache, path.dirname(bucket))
// we use @npmcli/move-file directly here because we
// want to overwrite the existing file
await moveFile(tmp.target, bucket)
tmp.moved = true
try {
await fixOwner.chownr(cache, bucket)
} catch (err) {
if (err.code !== 'ENOENT')
throw err
}
}
// write the file atomically
await disposer(setup(), teardown, write)
// we reverse the list we generated such that the newest
// entries come first in order to make looping through them easier
// the true passed to formatEntry tells it to keep null
// integrity values, if they made it this far it's because
// validateEntry returned true, and as such we should return it
return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))
}
module.exports.insert = insert
function insert (cache, key, integrity, opts) {
opts = IndexOpts(opts)
function insert (cache, key, integrity, opts = {}) {
const { metadata, size } = opts
const bucket = bucketPath(cache, key)
const entry = {
key,
integrity: integrity && ssri.stringify(integrity),
time: Date.now(),
size: opts.size,
metadata: opts.metadata
size,
metadata,
}
return fixOwner
.mkdirfix(cache, path.dirname(bucket))
@@ -53,14 +130,15 @@ function insert (cache, key, integrity, opts) {
// another while still preserving the string length of the JSON in
// question. So, we just slap the length in there and verify it on read.
//
// Thanks to @isaacs for the whiteboarding session that ended up with this.
// Thanks to @isaacs for the whiteboarding session that ended up with
// this.
return appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
})
.then(() => fixOwner.chownr(cache, bucket))
.catch((err) => {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return undefined
}
throw err
// There's a class of race conditions that happen when things get deleted
// during fixOwner, or between the two mkdirfix/chownr calls.
@@ -75,15 +153,15 @@ function insert (cache, key, integrity, opts) {
module.exports.insert.sync = insertSync
function insertSync (cache, key, integrity, opts) {
opts = IndexOpts(opts)
function insertSync (cache, key, integrity, opts = {}) {
const { metadata, size } = opts
const bucket = bucketPath(cache, key)
const entry = {
key,
integrity: integrity && ssri.stringify(integrity),
time: Date.now(),
size: opts.size,
metadata: opts.metadata
size,
metadata,
}
fixOwner.mkdirfix.sync(cache, path.dirname(bucket))
const stringified = JSON.stringify(entry)
@@ -91,9 +169,8 @@ function insertSync (cache, key, integrity, opts) {
try {
fixOwner.chownr.sync(cache, bucket)
} catch (err) {
if (err.code !== 'ENOENT') {
if (err.code !== 'ENOENT')
throw err
}
}
return formatEntry(cache, entry)
}
@@ -105,19 +182,17 @@ function find (cache, key) {
return bucketEntries(bucket)
.then((entries) => {
return entries.reduce((latest, next) => {
if (next && next.key === key) {
if (next && next.key === key)
return formatEntry(cache, next)
} else {
else
return latest
}
}, null)
})
.catch((err) => {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return null
} else {
else
throw err
}
})
}
@@ -127,31 +202,37 @@ function findSync (cache, key) {
const bucket = bucketPath(cache, key)
try {
return bucketEntriesSync(bucket).reduce((latest, next) => {
if (next && next.key === key) {
if (next && next.key === key)
return formatEntry(cache, next)
} else {
else
return latest
}
}, null)
} catch (err) {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return null
} else {
else
throw err
}
}
}
module.exports.delete = del
function del (cache, key, opts) {
return insert(cache, key, null, opts)
function del (cache, key, opts = {}) {
if (!opts.removeFully)
return insert(cache, key, null, opts)
const bucket = bucketPath(cache, key)
return rimraf(bucket)
}
module.exports.delete.sync = delSync
function delSync (cache, key, opts) {
return insertSync(cache, key, null, opts)
function delSync (cache, key, opts = {}) {
if (!opts.removeFully)
return insertSync(cache, key, null, opts)
const bucket = bucketPath(cache, key)
return rimraf.sync(bucket)
}
module.exports.lsStream = lsStream
@@ -182,12 +263,12 @@ function lsStream (cache) {
// reduced is a map of key => entry
for (const entry of reduced.values()) {
const formatted = formatEntry(cache, entry)
if (formatted) {
if (formatted)
stream.write(formatted)
}
}
}).catch(err => {
if (err.code === 'ENOENT') { return undefined }
if (err.code === 'ENOENT')
return undefined
throw err
})
})
@@ -215,10 +296,14 @@ function ls (cache) {
)
}
module.exports.bucketEntries = bucketEntries
function bucketEntries (bucket, filter) {
return readFile(bucket, 'utf8').then((data) => _bucketEntries(data, filter))
}
module.exports.bucketEntries.sync = bucketEntriesSync
function bucketEntriesSync (bucket, filter) {
const data = fs.readFileSync(bucket, 'utf8')
return _bucketEntries(data, filter)
@@ -227,9 +312,9 @@ function bucketEntriesSync (bucket, filter) {
function _bucketEntries (data, filter) {
const entries = []
data.split('\n').forEach((entry) => {
if (!entry) {
if (!entry)
return
}
const pieces = entry.split('\t')
if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
// Hash is no good! Corruption or malice? Doesn't matter!
@@ -243,9 +328,8 @@ function _bucketEntries (data, filter) {
// Entry is corrupted!
return
}
if (obj) {
if (obj)
entries.push(obj)
}
})
return entries
}
@@ -285,26 +369,25 @@ function hash (str, digest) {
.digest('hex')
}
function formatEntry (cache, entry) {
function formatEntry (cache, entry, keepAll) {
// Treat null digests as deletions. They'll shadow any previous entries.
if (!entry.integrity) {
if (!entry.integrity && !keepAll)
return null
}
return {
key: entry.key,
integrity: entry.integrity,
path: contentPath(cache, entry.integrity),
path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,
size: entry.size,
time: entry.time,
metadata: entry.metadata
metadata: entry.metadata,
}
}
function readdirOrEmpty (dir) {
return readdir(dir).catch((err) => {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
return []
}
throw err
})
+5 -6
View File
@@ -8,7 +8,7 @@ const MAX_AGE = 3 * 60 * 1000
const MEMOIZED = new LRU({
max: MAX_SIZE,
maxAge: MAX_AGE,
length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length
length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length,
})
module.exports.clearMemoized = clearMemoized
@@ -62,13 +62,12 @@ class ObjProxy {
}
function pickMem (opts) {
if (!opts || !opts.memoize) {
if (!opts || !opts.memoize)
return MEMOIZED
} else if (opts.memoize.get && opts.memoize.set) {
else if (opts.memoize.get && opts.memoize.set)
return opts.memoize
} else if (typeof opts.memoize === 'object') {
else if (typeof opts.memoize === 'object')
return new ObjProxy(opts.memoize)
} else {
else
return MEMOIZED
}
}
+2 -2
View File
@@ -8,9 +8,9 @@ function disposer (creatorFn, disposerFn, fn) {
.then(
// disposer resolved, do something with original fn's promise
() => {
if (shouldThrow) {
if (shouldThrow)
throw result
}
return result
},
// Disposer fn failed, crash process
+10 -13
View File
@@ -3,7 +3,7 @@
const util = require('util')
const chownr = util.promisify(require('chownr'))
const mkdirp = util.promisify(require('mkdirp'))
const mkdirp = require('mkdirp')
const inflight = require('promise-inflight')
const inferOwner = require('infer-owner')
@@ -49,9 +49,8 @@ function fixOwner (cache, filepath) {
const { uid, gid } = owner
// No need to override if it's already what we used.
if (self.uid === uid && self.gid === gid) {
if (self.uid === uid && self.gid === gid)
return
}
return inflight('fixOwner: fixing ownership on ' + filepath, () =>
chownr(
@@ -59,9 +58,9 @@ function fixOwner (cache, filepath) {
typeof uid === 'number' ? uid : self.uid,
typeof gid === 'number' ? gid : self.gid
).catch((err) => {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return null
}
throw err
})
)
@@ -94,9 +93,9 @@ function fixOwnerSync (cache, filepath) {
)
} catch (err) {
// only catch ENOENT, any other error is a problem.
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return null
}
throw err
}
}
@@ -111,14 +110,13 @@ function mkdirfix (cache, p, cb) {
return Promise.resolve(inferOwner(cache)).then(() => {
return mkdirp(p)
.then((made) => {
if (made) {
if (made)
return fixOwner(cache, made).then(() => made)
}
})
.catch((err) => {
if (err.code === 'EEXIST') {
if (err.code === 'EEXIST')
return fixOwner(cache, p).then(() => null)
}
throw err
})
})
@@ -138,8 +136,7 @@ function mkdirfixSync (cache, p) {
if (err.code === 'EEXIST') {
fixOwnerSync(cache, p)
return null
} else {
} else
throw err
}
}
}
+22 -11
View File
@@ -1,16 +1,19 @@
'use strict'
const fs = require('graceful-fs')
const fs = require('fs')
const util = require('util')
const chmod = util.promisify(fs.chmod)
const unlink = util.promisify(fs.unlink)
const stat = util.promisify(fs.stat)
const move = require('move-concurrently')
const move = require('@npmcli/move-file')
const pinflight = require('promise-inflight')
module.exports = moveFile
function moveFile (src, dest) {
const isWindows = global.__CACACHE_TEST_FAKE_WINDOWS__ ||
process.platform === 'win32'
// This isn't quite an fs.rename -- the assumption is that
// if `dest` already exists, and we get certain errors while
// trying to move it, we should just not bother.
@@ -23,22 +26,29 @@ function moveFile (src, dest) {
return new Promise((resolve, reject) => {
fs.link(src, dest, (err) => {
if (err) {
if (err.code === 'EEXIST' || err.code === 'EBUSY') {
if (isWindows && err.code === 'EPERM') {
// XXX This is a really weird way to handle this situation, as it
// results in the src file being deleted even though the dest
// might not exist. Since we pretty much always write files to
// deterministic locations based on content hash, this is likely
// ok (or at worst, just ends in a future cache miss). But it would
// be worth investigating at some time in the future if this is
// really what we want to do here.
return resolve()
} else if (err.code === 'EEXIST' || err.code === 'EBUSY') {
// file already exists, so whatever
} else if (err.code === 'EPERM' && process.platform === 'win32') {
// file handle stayed open even past graceful-fs limits
} else {
return resolve()
} else
return reject(err)
}
}
return resolve()
} else
return resolve()
})
})
.then(() => {
// content should never change for any reason, so make it read-only
return Promise.all([
unlink(src),
process.platform !== 'win32' && chmod(dest, '0444')
!isWindows && chmod(dest, '0444'),
])
})
.catch(() => {
@@ -49,7 +59,8 @@ function moveFile (src, dest) {
throw err
}
// file doesn't already exist! let's try a rename -> copy fallback
return move(src, dest, { Promise, fs })
// only delete if it successfully copies
return move(src, dest)
})
})
})
+12 -19
View File
@@ -1,26 +1,21 @@
'use strict'
const util = require('util')
const fs = require('@npmcli/fs')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./fix-owner')
const path = require('path')
const rimraf = util.promisify(require('rimraf'))
const uniqueFilename = require('unique-filename')
const { disposer } = require('./disposer')
const TmpOpts = figgyPudding({
tmpPrefix: {}
})
module.exports.mkdir = mktmpdir
function mktmpdir (cache, opts) {
opts = TmpOpts(opts)
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
return fixOwner.mkdirfix(cache, tmpTarget).then(() => {
return tmpTarget
})
function mktmpdir (cache, opts = {}) {
const { tmpPrefix } = opts
const tmpDir = path.join(cache, 'tmp')
return fs.mkdir(tmpDir, { recursive: true, owner: 'inherit' })
.then(() => {
// do not use path.join(), it drops the trailing / if tmpPrefix is unset
const target = `${tmpDir}${path.sep}${tmpPrefix || ''}`
return fs.mkdtemp(target, { owner: 'inherit' })
})
}
module.exports.withTmp = withTmp
@@ -28,11 +23,9 @@ module.exports.withTmp = withTmp
function withTmp (cache, opts, cb) {
if (!cb) {
cb = opts
opts = null
opts = {}
}
opts = TmpOpts(opts)
return disposer(mktmpdir(cache, opts), rimraf, cb)
return fs.withTempDir(path.join(cache, 'tmp'), cb, opts)
}
module.exports.fix = fixtmpdir
+24 -28
View File
@@ -4,9 +4,8 @@ const util = require('util')
const pMap = require('p-map')
const contentPath = require('./content/path')
const figgyPudding = require('figgy-pudding')
const fixOwner = require('./util/fix-owner')
const fs = require('graceful-fs')
const fs = require('fs')
const fsm = require('fs-minipass')
const glob = util.promisify(require('glob'))
const index = require('./entry-index')
@@ -22,20 +21,16 @@ const truncate = util.promisify(fs.truncate)
const writeFile = util.promisify(fs.writeFile)
const readFile = util.promisify(fs.readFile)
const VerifyOpts = figgyPudding({
concurrency: {
default: 20
},
filter: {},
log: {
default: { silly () {} }
}
const verifyOpts = (opts) => ({
concurrency: 20,
log: { silly () {} },
...opts,
})
module.exports = verify
function verify (cache, opts) {
opts = VerifyOpts(opts)
opts = verifyOpts(opts)
opts.log.silly('verify', 'verifying cache at', cache)
const steps = [
@@ -45,12 +40,12 @@ function verify (cache, opts) {
rebuildIndex,
cleanTmp,
writeVerifile,
markEndTime
markEndTime,
]
return steps
.reduce((promise, step, i) => {
const label = step.name || `step #${i}`
const label = step.name
const start = new Date()
return promise.then((stats) => {
return step(cache, opts).then((s) => {
@@ -59,9 +54,9 @@ function verify (cache, opts) {
stats[k] = s[k]
})
const end = new Date()
if (!stats.runTime) {
if (!stats.runTime)
stats.runTime = {}
}
stats.runTime[label] = end - start
return Promise.resolve(stats)
})
@@ -113,9 +108,9 @@ function garbageCollect (cache, opts) {
const indexStream = index.lsStream(cache)
const liveContent = new Set()
indexStream.on('data', (entry) => {
if (opts.filter && !opts.filter(entry)) {
if (opts.filter && !opts.filter(entry))
return
}
liveContent.add(entry.integrity.toString())
})
return new Promise((resolve, reject) => {
@@ -125,14 +120,14 @@ function garbageCollect (cache, opts) {
return glob(path.join(contentDir, '**'), {
follow: false,
nodir: true,
nosort: true
nosort: true,
}).then((files) => {
return Promise.resolve({
verifiedContent: 0,
reclaimedCount: 0,
reclaimedSize: 0,
badContentCount: 0,
keptSize: 0
keptSize: 0,
}).then((stats) =>
pMap(
files,
@@ -176,14 +171,14 @@ function verifyContent (filepath, sri) {
.then((s) => {
const contentInfo = {
size: s.size,
valid: true
valid: true,
}
return ssri
.checkStream(new fsm.ReadStream(filepath), sri)
.catch((err) => {
if (err.code !== 'EINTEGRITY') {
if (err.code !== 'EINTEGRITY')
throw err
}
return rimraf(filepath).then(() => {
contentInfo.valid = false
})
@@ -191,9 +186,9 @@ function verifyContent (filepath, sri) {
.then(() => contentInfo)
})
.catch((err) => {
if (err.code === 'ENOENT') {
if (err.code === 'ENOENT')
return { size: 0, valid: false }
}
throw err
})
}
@@ -204,18 +199,19 @@ function rebuildIndex (cache, opts) {
const stats = {
missingContent: 0,
rejectedEntries: 0,
totalEntries: 0
totalEntries: 0,
}
const buckets = {}
for (const k in entries) {
/* istanbul ignore else */
if (hasOwnProperty(entries, k)) {
const hashed = index.hashKey(k)
const entry = entries[k]
const excluded = opts.filter && !opts.filter(entry)
excluded && stats.rejectedEntries++
if (buckets[hashed] && !excluded) {
if (buckets[hashed] && !excluded)
buckets[hashed].push(entry)
} else if (buckets[hashed] && excluded) {
else if (buckets[hashed] && excluded) {
// skip
} else if (excluded) {
buckets[hashed] = []
@@ -248,7 +244,7 @@ function rebuildBucket (cache, bucket, stats, opts) {
return index
.insert(cache, entry.key, entry.integrity, {
metadata: entry.metadata,
size: entry.size
size: entry.size,
})
.then(() => {
stats.totalEntries++