forked from daren.hsu/line_push
update
This commit is contained in:
+4
-3
@@ -15,9 +15,10 @@ module.exports = contentPath
|
||||
function contentPath (cache, integrity) {
|
||||
const sri = ssri.parse(integrity, { single: true })
|
||||
// contentPath is the *strongest* algo given
|
||||
return path.join.apply(
|
||||
path,
|
||||
[contentDir(cache), sri.algorithm].concat(hashToSegments(sri.hexDigest()))
|
||||
return path.join(
|
||||
contentDir(cache),
|
||||
sri.algorithm,
|
||||
...hashToSegments(sri.hexDigest())
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
+45
-62
@@ -2,8 +2,7 @@
|
||||
|
||||
const util = require('util')
|
||||
|
||||
const figgyPudding = require('figgy-pudding')
|
||||
const fs = require('graceful-fs')
|
||||
const fs = require('fs')
|
||||
const fsm = require('fs-minipass')
|
||||
const ssri = require('ssri')
|
||||
const contentPath = require('./path')
|
||||
@@ -12,30 +11,25 @@ const Pipeline = require('minipass-pipeline')
|
||||
const lstat = util.promisify(fs.lstat)
|
||||
const readFile = util.promisify(fs.readFile)
|
||||
|
||||
const ReadOpts = figgyPudding({
|
||||
size: {}
|
||||
})
|
||||
|
||||
module.exports = read
|
||||
|
||||
const MAX_SINGLE_READ_SIZE = 64 * 1024 * 1024
|
||||
function read (cache, integrity, opts) {
|
||||
opts = ReadOpts(opts)
|
||||
function read (cache, integrity, opts = {}) {
|
||||
const { size } = opts
|
||||
return withContentSri(cache, integrity, (cpath, sri) => {
|
||||
// get size
|
||||
return lstat(cpath).then(stat => ({ stat, cpath, sri }))
|
||||
}).then(({ stat, cpath, sri }) => {
|
||||
if (typeof opts.size === 'number' && stat.size !== opts.size) {
|
||||
throw sizeError(opts.size, stat.size)
|
||||
}
|
||||
if (stat.size > MAX_SINGLE_READ_SIZE) {
|
||||
if (typeof size === 'number' && stat.size !== size)
|
||||
throw sizeError(size, stat.size)
|
||||
|
||||
if (stat.size > MAX_SINGLE_READ_SIZE)
|
||||
return readPipeline(cpath, stat.size, sri, new Pipeline()).concat()
|
||||
}
|
||||
|
||||
return readFile(cpath, null).then((data) => {
|
||||
if (!ssri.checkData(data, sri)) {
|
||||
if (!ssri.checkData(data, sri))
|
||||
throw integrityError(sri, cpath)
|
||||
}
|
||||
|
||||
return data
|
||||
})
|
||||
})
|
||||
@@ -45,11 +39,11 @@ const readPipeline = (cpath, size, sri, stream) => {
|
||||
stream.push(
|
||||
new fsm.ReadStream(cpath, {
|
||||
size,
|
||||
readSize: MAX_SINGLE_READ_SIZE
|
||||
readSize: MAX_SINGLE_READ_SIZE,
|
||||
}),
|
||||
ssri.integrityStream({
|
||||
integrity: sri,
|
||||
size
|
||||
size,
|
||||
})
|
||||
)
|
||||
return stream
|
||||
@@ -57,17 +51,15 @@ const readPipeline = (cpath, size, sri, stream) => {
|
||||
|
||||
module.exports.sync = readSync
|
||||
|
||||
function readSync (cache, integrity, opts) {
|
||||
opts = ReadOpts(opts)
|
||||
function readSync (cache, integrity, opts = {}) {
|
||||
const { size } = opts
|
||||
return withContentSriSync(cache, integrity, (cpath, sri) => {
|
||||
const data = fs.readFileSync(cpath)
|
||||
if (typeof opts.size === 'number' && opts.size !== data.length) {
|
||||
throw sizeError(opts.size, data.length)
|
||||
}
|
||||
if (typeof size === 'number' && size !== data.length)
|
||||
throw sizeError(size, data.length)
|
||||
|
||||
if (ssri.checkData(data, sri)) {
|
||||
if (ssri.checkData(data, sri))
|
||||
return data
|
||||
}
|
||||
|
||||
throw integrityError(sri, cpath)
|
||||
})
|
||||
@@ -76,17 +68,16 @@ function readSync (cache, integrity, opts) {
|
||||
module.exports.stream = readStream
|
||||
module.exports.readStream = readStream
|
||||
|
||||
function readStream (cache, integrity, opts) {
|
||||
opts = ReadOpts(opts)
|
||||
|
||||
function readStream (cache, integrity, opts = {}) {
|
||||
const { size } = opts
|
||||
const stream = new Pipeline()
|
||||
withContentSri(cache, integrity, (cpath, sri) => {
|
||||
// just lstat to ensure it exists
|
||||
return lstat(cpath).then((stat) => ({ stat, cpath, sri }))
|
||||
}).then(({ stat, cpath, sri }) => {
|
||||
if (typeof opts.size === 'number' && opts.size !== stat.size) {
|
||||
return stream.emit('error', sizeError(opts.size, stat.size))
|
||||
}
|
||||
if (typeof size === 'number' && size !== stat.size)
|
||||
return stream.emit('error', sizeError(size, stat.size))
|
||||
|
||||
readPipeline(cpath, stat.size, sri, stream)
|
||||
}, er => stream.emit('error', er))
|
||||
|
||||
@@ -100,15 +91,13 @@ if (fs.copyFile) {
|
||||
copyFile = util.promisify(fs.copyFile)
|
||||
}
|
||||
|
||||
function copy (cache, integrity, dest, opts) {
|
||||
opts = ReadOpts(opts)
|
||||
function copy (cache, integrity, dest) {
|
||||
return withContentSri(cache, integrity, (cpath, sri) => {
|
||||
return copyFile(cpath, dest)
|
||||
})
|
||||
}
|
||||
|
||||
function copySync (cache, integrity, dest, opts) {
|
||||
opts = ReadOpts(opts)
|
||||
function copySync (cache, integrity, dest) {
|
||||
return withContentSriSync(cache, integrity, (cpath, sri) => {
|
||||
return fs.copyFileSync(cpath, dest)
|
||||
})
|
||||
@@ -117,21 +106,21 @@ function copySync (cache, integrity, dest, opts) {
|
||||
module.exports.hasContent = hasContent
|
||||
|
||||
function hasContent (cache, integrity) {
|
||||
if (!integrity) {
|
||||
if (!integrity)
|
||||
return Promise.resolve(false)
|
||||
}
|
||||
|
||||
return withContentSri(cache, integrity, (cpath, sri) => {
|
||||
return lstat(cpath).then((stat) => ({ size: stat.size, sri, stat }))
|
||||
}).catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return false
|
||||
}
|
||||
|
||||
if (err.code === 'EPERM') {
|
||||
if (process.platform !== 'win32') {
|
||||
/* istanbul ignore else */
|
||||
if (process.platform !== 'win32')
|
||||
throw err
|
||||
} else {
|
||||
else
|
||||
return false
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -139,23 +128,23 @@ function hasContent (cache, integrity) {
|
||||
module.exports.hasContent.sync = hasContentSync
|
||||
|
||||
function hasContentSync (cache, integrity) {
|
||||
if (!integrity) {
|
||||
if (!integrity)
|
||||
return false
|
||||
}
|
||||
|
||||
return withContentSriSync(cache, integrity, (cpath, sri) => {
|
||||
try {
|
||||
const stat = fs.lstatSync(cpath)
|
||||
return { size: stat.size, sri, stat }
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return false
|
||||
}
|
||||
|
||||
if (err.code === 'EPERM') {
|
||||
if (process.platform !== 'win32') {
|
||||
/* istanbul ignore else */
|
||||
if (process.platform !== 'win32')
|
||||
throw err
|
||||
} else {
|
||||
else
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -173,9 +162,10 @@ function withContentSri (cache, integrity, fn) {
|
||||
const cpath = contentPath(cache, digests[0])
|
||||
return fn(cpath, digests[0])
|
||||
} else {
|
||||
// Can't use race here because a generic error can happen before a ENOENT error, and can happen before a valid result
|
||||
// Can't use race here because a generic error can happen before
|
||||
// a ENOENT error, and can happen before a valid result
|
||||
return Promise
|
||||
.all(sri[sri.pickAlgorithm()].map((meta) => {
|
||||
.all(digests.map((meta) => {
|
||||
return withContentSri(cache, meta, fn)
|
||||
.catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
@@ -190,21 +180,16 @@ function withContentSri (cache, integrity, fn) {
|
||||
.then((results) => {
|
||||
// Return the first non error if it is found
|
||||
const result = results.find((r) => !(r instanceof Error))
|
||||
if (result) {
|
||||
if (result)
|
||||
return result
|
||||
}
|
||||
|
||||
// Throw the No matching content found error
|
||||
const enoentError = results.find((r) => r.code === 'ENOENT')
|
||||
if (enoentError) {
|
||||
if (enoentError)
|
||||
throw enoentError
|
||||
}
|
||||
|
||||
// Throw generic error
|
||||
const genericError = results.find((r) => r instanceof Error)
|
||||
if (genericError) {
|
||||
throw genericError
|
||||
}
|
||||
throw results.find((r) => r instanceof Error)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -231,16 +216,14 @@ function withContentSriSync (cache, integrity, fn) {
|
||||
return fn(cpath, digests[0])
|
||||
} else {
|
||||
let lastErr = null
|
||||
for (const meta of sri[sri.pickAlgorithm()]) {
|
||||
for (const meta of digests) {
|
||||
try {
|
||||
return withContentSriSync(cache, meta, fn)
|
||||
} catch (err) {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
if (lastErr) {
|
||||
throw lastErr
|
||||
}
|
||||
throw lastErr
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
+4
-7
@@ -10,13 +10,10 @@ module.exports = rm
|
||||
|
||||
function rm (cache, integrity) {
|
||||
return hasContent(cache, integrity).then((content) => {
|
||||
if (content) {
|
||||
const sri = content.sri
|
||||
if (sri) {
|
||||
return rimraf(contentPath(cache, sri)).then(() => true)
|
||||
}
|
||||
} else {
|
||||
// ~pretty~ sure we can't end up with a content lacking sri, but be safe
|
||||
if (content && content.sri)
|
||||
return rimraf(contentPath(cache, content.sri)).then(() => true)
|
||||
else
|
||||
return false
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
+26
-24
@@ -4,7 +4,7 @@ const util = require('util')
|
||||
|
||||
const contentPath = require('./path')
|
||||
const fixOwner = require('../util/fix-owner')
|
||||
const fs = require('graceful-fs')
|
||||
const fs = require('fs')
|
||||
const moveFile = require('../util/move-file')
|
||||
const Minipass = require('minipass')
|
||||
const Pipeline = require('minipass-pipeline')
|
||||
@@ -20,20 +20,17 @@ const writeFile = util.promisify(fs.writeFile)
|
||||
|
||||
module.exports = write
|
||||
|
||||
function write (cache, data, opts) {
|
||||
opts = opts || {}
|
||||
if (opts.algorithms && opts.algorithms.length > 1) {
|
||||
function write (cache, data, opts = {}) {
|
||||
const { algorithms, size, integrity } = opts
|
||||
if (algorithms && algorithms.length > 1)
|
||||
throw new Error('opts.algorithms only supports a single algorithm for now')
|
||||
}
|
||||
if (typeof opts.size === 'number' && data.length !== opts.size) {
|
||||
return Promise.reject(sizeError(opts.size, data.length))
|
||||
}
|
||||
const sri = ssri.fromData(data, {
|
||||
algorithms: opts.algorithms
|
||||
})
|
||||
if (opts.integrity && !ssri.checkData(data, opts.integrity, opts)) {
|
||||
return Promise.reject(checksumError(opts.integrity, sri))
|
||||
}
|
||||
|
||||
if (typeof size === 'number' && data.length !== size)
|
||||
return Promise.reject(sizeError(size, data.length))
|
||||
|
||||
const sri = ssri.fromData(data, algorithms ? { algorithms } : {})
|
||||
if (integrity && !ssri.checkData(data, integrity, opts))
|
||||
return Promise.reject(checksumError(integrity, sri))
|
||||
|
||||
return disposer(makeTmp(cache, opts), makeTmpDisposer,
|
||||
(tmp) => {
|
||||
@@ -90,8 +87,7 @@ class CacacheWriteStream extends Flush {
|
||||
}
|
||||
}
|
||||
|
||||
function writeStream (cache, opts) {
|
||||
opts = opts || {}
|
||||
function writeStream (cache, opts = {}) {
|
||||
return new CacacheWriteStream(cache, opts)
|
||||
}
|
||||
|
||||
@@ -115,13 +111,17 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
|
||||
const hashStream = ssri.integrityStream({
|
||||
integrity: opts.integrity,
|
||||
algorithms: opts.algorithms,
|
||||
size: opts.size
|
||||
size: opts.size,
|
||||
})
|
||||
hashStream.on('integrity', i => {
|
||||
integrity = i
|
||||
})
|
||||
hashStream.on('size', s => {
|
||||
size = s
|
||||
})
|
||||
hashStream.on('integrity', i => { integrity = i })
|
||||
hashStream.on('size', s => { size = s })
|
||||
|
||||
const outStream = new fsm.WriteStream(tmpTarget, {
|
||||
flags: 'wx'
|
||||
flags: 'wx',
|
||||
})
|
||||
|
||||
// NB: this can throw if the hashStream has a problem with
|
||||
@@ -135,21 +135,23 @@ function pipeToTmp (inputStream, cache, tmpTarget, opts) {
|
||||
|
||||
return pipeline.promise()
|
||||
.then(() => ({ integrity, size }))
|
||||
.catch(er => rimraf(tmpTarget).then(() => { throw er }))
|
||||
.catch(er => rimraf(tmpTarget).then(() => {
|
||||
throw er
|
||||
}))
|
||||
}
|
||||
|
||||
function makeTmp (cache, opts) {
|
||||
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
|
||||
return fixOwner.mkdirfix(cache, path.dirname(tmpTarget)).then(() => ({
|
||||
target: tmpTarget,
|
||||
moved: false
|
||||
moved: false,
|
||||
}))
|
||||
}
|
||||
|
||||
function makeTmpDisposer (tmp) {
|
||||
if (tmp.moved) {
|
||||
if (tmp.moved)
|
||||
return Promise.resolve()
|
||||
}
|
||||
|
||||
return rimraf(tmp.target)
|
||||
}
|
||||
|
||||
|
||||
+133
-50
@@ -1,21 +1,27 @@
|
||||
'use strict'
|
||||
|
||||
const util = require('util')
|
||||
|
||||
const crypto = require('crypto')
|
||||
const figgyPudding = require('figgy-pudding')
|
||||
const fs = require('graceful-fs')
|
||||
const fs = require('fs')
|
||||
const Minipass = require('minipass')
|
||||
const path = require('path')
|
||||
const ssri = require('ssri')
|
||||
const uniqueFilename = require('unique-filename')
|
||||
|
||||
const { disposer } = require('./util/disposer')
|
||||
const contentPath = require('./content/path')
|
||||
const fixOwner = require('./util/fix-owner')
|
||||
const hashToSegments = require('./util/hash-to-segments')
|
||||
const indexV = require('../package.json')['cache-version'].index
|
||||
const moveFile = require('@npmcli/move-file')
|
||||
const _rimraf = require('rimraf')
|
||||
const rimraf = util.promisify(_rimraf)
|
||||
rimraf.sync = _rimraf.sync
|
||||
|
||||
const appendFile = util.promisify(fs.appendFile)
|
||||
const readFile = util.promisify(fs.readFile)
|
||||
const readdir = util.promisify(fs.readdir)
|
||||
const writeFile = util.promisify(fs.writeFile)
|
||||
|
||||
module.exports.NotFoundError = class NotFoundError extends Error {
|
||||
constructor (cache, key) {
|
||||
@@ -26,22 +32,93 @@ module.exports.NotFoundError = class NotFoundError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
const IndexOpts = figgyPudding({
|
||||
metadata: {},
|
||||
size: {}
|
||||
})
|
||||
module.exports.compact = compact
|
||||
|
||||
async function compact (cache, key, matchFn, opts = {}) {
|
||||
const bucket = bucketPath(cache, key)
|
||||
const entries = await bucketEntries(bucket)
|
||||
const newEntries = []
|
||||
// we loop backwards because the bottom-most result is the newest
|
||||
// since we add new entries with appendFile
|
||||
for (let i = entries.length - 1; i >= 0; --i) {
|
||||
const entry = entries[i]
|
||||
// a null integrity could mean either a delete was appended
|
||||
// or the user has simply stored an index that does not map
|
||||
// to any content. we determine if the user wants to keep the
|
||||
// null integrity based on the validateEntry function passed in options.
|
||||
// if the integrity is null and no validateEntry is provided, we break
|
||||
// as we consider the null integrity to be a deletion of everything
|
||||
// that came before it.
|
||||
if (entry.integrity === null && !opts.validateEntry)
|
||||
break
|
||||
|
||||
// if this entry is valid, and it is either the first entry or
|
||||
// the newEntries array doesn't already include an entry that
|
||||
// matches this one based on the provided matchFn, then we add
|
||||
// it to the beginning of our list
|
||||
if ((!opts.validateEntry || opts.validateEntry(entry) === true) &&
|
||||
(newEntries.length === 0 ||
|
||||
!newEntries.find((oldEntry) => matchFn(oldEntry, entry))))
|
||||
newEntries.unshift(entry)
|
||||
}
|
||||
|
||||
const newIndex = '\n' + newEntries.map((entry) => {
|
||||
const stringified = JSON.stringify(entry)
|
||||
const hash = hashEntry(stringified)
|
||||
return `${hash}\t${stringified}`
|
||||
}).join('\n')
|
||||
|
||||
const setup = async () => {
|
||||
const target = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
|
||||
await fixOwner.mkdirfix(cache, path.dirname(target))
|
||||
return {
|
||||
target,
|
||||
moved: false,
|
||||
}
|
||||
}
|
||||
|
||||
const teardown = async (tmp) => {
|
||||
if (!tmp.moved)
|
||||
return rimraf(tmp.target)
|
||||
}
|
||||
|
||||
const write = async (tmp) => {
|
||||
await writeFile(tmp.target, newIndex, { flag: 'wx' })
|
||||
await fixOwner.mkdirfix(cache, path.dirname(bucket))
|
||||
// we use @npmcli/move-file directly here because we
|
||||
// want to overwrite the existing file
|
||||
await moveFile(tmp.target, bucket)
|
||||
tmp.moved = true
|
||||
try {
|
||||
await fixOwner.chownr(cache, bucket)
|
||||
} catch (err) {
|
||||
if (err.code !== 'ENOENT')
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
// write the file atomically
|
||||
await disposer(setup(), teardown, write)
|
||||
|
||||
// we reverse the list we generated such that the newest
|
||||
// entries come first in order to make looping through them easier
|
||||
// the true passed to formatEntry tells it to keep null
|
||||
// integrity values, if they made it this far it's because
|
||||
// validateEntry returned true, and as such we should return it
|
||||
return newEntries.reverse().map((entry) => formatEntry(cache, entry, true))
|
||||
}
|
||||
|
||||
module.exports.insert = insert
|
||||
|
||||
function insert (cache, key, integrity, opts) {
|
||||
opts = IndexOpts(opts)
|
||||
function insert (cache, key, integrity, opts = {}) {
|
||||
const { metadata, size } = opts
|
||||
const bucket = bucketPath(cache, key)
|
||||
const entry = {
|
||||
key,
|
||||
integrity: integrity && ssri.stringify(integrity),
|
||||
time: Date.now(),
|
||||
size: opts.size,
|
||||
metadata: opts.metadata
|
||||
size,
|
||||
metadata,
|
||||
}
|
||||
return fixOwner
|
||||
.mkdirfix(cache, path.dirname(bucket))
|
||||
@@ -53,14 +130,15 @@ function insert (cache, key, integrity, opts) {
|
||||
// another while still preserving the string length of the JSON in
|
||||
// question. So, we just slap the length in there and verify it on read.
|
||||
//
|
||||
// Thanks to @isaacs for the whiteboarding session that ended up with this.
|
||||
// Thanks to @isaacs for the whiteboarding session that ended up with
|
||||
// this.
|
||||
return appendFile(bucket, `\n${hashEntry(stringified)}\t${stringified}`)
|
||||
})
|
||||
.then(() => fixOwner.chownr(cache, bucket))
|
||||
.catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return undefined
|
||||
}
|
||||
|
||||
throw err
|
||||
// There's a class of race conditions that happen when things get deleted
|
||||
// during fixOwner, or between the two mkdirfix/chownr calls.
|
||||
@@ -75,15 +153,15 @@ function insert (cache, key, integrity, opts) {
|
||||
|
||||
module.exports.insert.sync = insertSync
|
||||
|
||||
function insertSync (cache, key, integrity, opts) {
|
||||
opts = IndexOpts(opts)
|
||||
function insertSync (cache, key, integrity, opts = {}) {
|
||||
const { metadata, size } = opts
|
||||
const bucket = bucketPath(cache, key)
|
||||
const entry = {
|
||||
key,
|
||||
integrity: integrity && ssri.stringify(integrity),
|
||||
time: Date.now(),
|
||||
size: opts.size,
|
||||
metadata: opts.metadata
|
||||
size,
|
||||
metadata,
|
||||
}
|
||||
fixOwner.mkdirfix.sync(cache, path.dirname(bucket))
|
||||
const stringified = JSON.stringify(entry)
|
||||
@@ -91,9 +169,8 @@ function insertSync (cache, key, integrity, opts) {
|
||||
try {
|
||||
fixOwner.chownr.sync(cache, bucket)
|
||||
} catch (err) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
if (err.code !== 'ENOENT')
|
||||
throw err
|
||||
}
|
||||
}
|
||||
return formatEntry(cache, entry)
|
||||
}
|
||||
@@ -105,19 +182,17 @@ function find (cache, key) {
|
||||
return bucketEntries(bucket)
|
||||
.then((entries) => {
|
||||
return entries.reduce((latest, next) => {
|
||||
if (next && next.key === key) {
|
||||
if (next && next.key === key)
|
||||
return formatEntry(cache, next)
|
||||
} else {
|
||||
else
|
||||
return latest
|
||||
}
|
||||
}, null)
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return null
|
||||
} else {
|
||||
else
|
||||
throw err
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
@@ -127,31 +202,37 @@ function findSync (cache, key) {
|
||||
const bucket = bucketPath(cache, key)
|
||||
try {
|
||||
return bucketEntriesSync(bucket).reduce((latest, next) => {
|
||||
if (next && next.key === key) {
|
||||
if (next && next.key === key)
|
||||
return formatEntry(cache, next)
|
||||
} else {
|
||||
else
|
||||
return latest
|
||||
}
|
||||
}, null)
|
||||
} catch (err) {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return null
|
||||
} else {
|
||||
else
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports.delete = del
|
||||
|
||||
function del (cache, key, opts) {
|
||||
return insert(cache, key, null, opts)
|
||||
function del (cache, key, opts = {}) {
|
||||
if (!opts.removeFully)
|
||||
return insert(cache, key, null, opts)
|
||||
|
||||
const bucket = bucketPath(cache, key)
|
||||
return rimraf(bucket)
|
||||
}
|
||||
|
||||
module.exports.delete.sync = delSync
|
||||
|
||||
function delSync (cache, key, opts) {
|
||||
return insertSync(cache, key, null, opts)
|
||||
function delSync (cache, key, opts = {}) {
|
||||
if (!opts.removeFully)
|
||||
return insertSync(cache, key, null, opts)
|
||||
|
||||
const bucket = bucketPath(cache, key)
|
||||
return rimraf.sync(bucket)
|
||||
}
|
||||
|
||||
module.exports.lsStream = lsStream
|
||||
@@ -182,12 +263,12 @@ function lsStream (cache) {
|
||||
// reduced is a map of key => entry
|
||||
for (const entry of reduced.values()) {
|
||||
const formatted = formatEntry(cache, entry)
|
||||
if (formatted) {
|
||||
if (formatted)
|
||||
stream.write(formatted)
|
||||
}
|
||||
}
|
||||
}).catch(err => {
|
||||
if (err.code === 'ENOENT') { return undefined }
|
||||
if (err.code === 'ENOENT')
|
||||
return undefined
|
||||
throw err
|
||||
})
|
||||
})
|
||||
@@ -215,10 +296,14 @@ function ls (cache) {
|
||||
)
|
||||
}
|
||||
|
||||
module.exports.bucketEntries = bucketEntries
|
||||
|
||||
function bucketEntries (bucket, filter) {
|
||||
return readFile(bucket, 'utf8').then((data) => _bucketEntries(data, filter))
|
||||
}
|
||||
|
||||
module.exports.bucketEntries.sync = bucketEntriesSync
|
||||
|
||||
function bucketEntriesSync (bucket, filter) {
|
||||
const data = fs.readFileSync(bucket, 'utf8')
|
||||
return _bucketEntries(data, filter)
|
||||
@@ -227,9 +312,9 @@ function bucketEntriesSync (bucket, filter) {
|
||||
function _bucketEntries (data, filter) {
|
||||
const entries = []
|
||||
data.split('\n').forEach((entry) => {
|
||||
if (!entry) {
|
||||
if (!entry)
|
||||
return
|
||||
}
|
||||
|
||||
const pieces = entry.split('\t')
|
||||
if (!pieces[1] || hashEntry(pieces[1]) !== pieces[0]) {
|
||||
// Hash is no good! Corruption or malice? Doesn't matter!
|
||||
@@ -243,9 +328,8 @@ function _bucketEntries (data, filter) {
|
||||
// Entry is corrupted!
|
||||
return
|
||||
}
|
||||
if (obj) {
|
||||
if (obj)
|
||||
entries.push(obj)
|
||||
}
|
||||
})
|
||||
return entries
|
||||
}
|
||||
@@ -285,26 +369,25 @@ function hash (str, digest) {
|
||||
.digest('hex')
|
||||
}
|
||||
|
||||
function formatEntry (cache, entry) {
|
||||
function formatEntry (cache, entry, keepAll) {
|
||||
// Treat null digests as deletions. They'll shadow any previous entries.
|
||||
if (!entry.integrity) {
|
||||
if (!entry.integrity && !keepAll)
|
||||
return null
|
||||
}
|
||||
|
||||
return {
|
||||
key: entry.key,
|
||||
integrity: entry.integrity,
|
||||
path: contentPath(cache, entry.integrity),
|
||||
path: entry.integrity ? contentPath(cache, entry.integrity) : undefined,
|
||||
size: entry.size,
|
||||
time: entry.time,
|
||||
metadata: entry.metadata
|
||||
metadata: entry.metadata,
|
||||
}
|
||||
}
|
||||
|
||||
function readdirOrEmpty (dir) {
|
||||
return readdir(dir).catch((err) => {
|
||||
if (err.code === 'ENOENT' || err.code === 'ENOTDIR') {
|
||||
if (err.code === 'ENOENT' || err.code === 'ENOTDIR')
|
||||
return []
|
||||
}
|
||||
|
||||
throw err
|
||||
})
|
||||
|
||||
+5
-6
@@ -8,7 +8,7 @@ const MAX_AGE = 3 * 60 * 1000
|
||||
const MEMOIZED = new LRU({
|
||||
max: MAX_SIZE,
|
||||
maxAge: MAX_AGE,
|
||||
length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length
|
||||
length: (entry, key) => key.startsWith('key:') ? entry.data.length : entry.length,
|
||||
})
|
||||
|
||||
module.exports.clearMemoized = clearMemoized
|
||||
@@ -62,13 +62,12 @@ class ObjProxy {
|
||||
}
|
||||
|
||||
function pickMem (opts) {
|
||||
if (!opts || !opts.memoize) {
|
||||
if (!opts || !opts.memoize)
|
||||
return MEMOIZED
|
||||
} else if (opts.memoize.get && opts.memoize.set) {
|
||||
else if (opts.memoize.get && opts.memoize.set)
|
||||
return opts.memoize
|
||||
} else if (typeof opts.memoize === 'object') {
|
||||
else if (typeof opts.memoize === 'object')
|
||||
return new ObjProxy(opts.memoize)
|
||||
} else {
|
||||
else
|
||||
return MEMOIZED
|
||||
}
|
||||
}
|
||||
|
||||
+2
-2
@@ -8,9 +8,9 @@ function disposer (creatorFn, disposerFn, fn) {
|
||||
.then(
|
||||
// disposer resolved, do something with original fn's promise
|
||||
() => {
|
||||
if (shouldThrow) {
|
||||
if (shouldThrow)
|
||||
throw result
|
||||
}
|
||||
|
||||
return result
|
||||
},
|
||||
// Disposer fn failed, crash process
|
||||
|
||||
+10
-13
@@ -3,7 +3,7 @@
|
||||
const util = require('util')
|
||||
|
||||
const chownr = util.promisify(require('chownr'))
|
||||
const mkdirp = util.promisify(require('mkdirp'))
|
||||
const mkdirp = require('mkdirp')
|
||||
const inflight = require('promise-inflight')
|
||||
const inferOwner = require('infer-owner')
|
||||
|
||||
@@ -49,9 +49,8 @@ function fixOwner (cache, filepath) {
|
||||
const { uid, gid } = owner
|
||||
|
||||
// No need to override if it's already what we used.
|
||||
if (self.uid === uid && self.gid === gid) {
|
||||
if (self.uid === uid && self.gid === gid)
|
||||
return
|
||||
}
|
||||
|
||||
return inflight('fixOwner: fixing ownership on ' + filepath, () =>
|
||||
chownr(
|
||||
@@ -59,9 +58,9 @@ function fixOwner (cache, filepath) {
|
||||
typeof uid === 'number' ? uid : self.uid,
|
||||
typeof gid === 'number' ? gid : self.gid
|
||||
).catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return null
|
||||
}
|
||||
|
||||
throw err
|
||||
})
|
||||
)
|
||||
@@ -94,9 +93,9 @@ function fixOwnerSync (cache, filepath) {
|
||||
)
|
||||
} catch (err) {
|
||||
// only catch ENOENT, any other error is a problem.
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return null
|
||||
}
|
||||
|
||||
throw err
|
||||
}
|
||||
}
|
||||
@@ -111,14 +110,13 @@ function mkdirfix (cache, p, cb) {
|
||||
return Promise.resolve(inferOwner(cache)).then(() => {
|
||||
return mkdirp(p)
|
||||
.then((made) => {
|
||||
if (made) {
|
||||
if (made)
|
||||
return fixOwner(cache, made).then(() => made)
|
||||
}
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err.code === 'EEXIST') {
|
||||
if (err.code === 'EEXIST')
|
||||
return fixOwner(cache, p).then(() => null)
|
||||
}
|
||||
|
||||
throw err
|
||||
})
|
||||
})
|
||||
@@ -138,8 +136,7 @@ function mkdirfixSync (cache, p) {
|
||||
if (err.code === 'EEXIST') {
|
||||
fixOwnerSync(cache, p)
|
||||
return null
|
||||
} else {
|
||||
} else
|
||||
throw err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+22
-11
@@ -1,16 +1,19 @@
|
||||
'use strict'
|
||||
|
||||
const fs = require('graceful-fs')
|
||||
const fs = require('fs')
|
||||
const util = require('util')
|
||||
const chmod = util.promisify(fs.chmod)
|
||||
const unlink = util.promisify(fs.unlink)
|
||||
const stat = util.promisify(fs.stat)
|
||||
const move = require('move-concurrently')
|
||||
const move = require('@npmcli/move-file')
|
||||
const pinflight = require('promise-inflight')
|
||||
|
||||
module.exports = moveFile
|
||||
|
||||
function moveFile (src, dest) {
|
||||
const isWindows = global.__CACACHE_TEST_FAKE_WINDOWS__ ||
|
||||
process.platform === 'win32'
|
||||
|
||||
// This isn't quite an fs.rename -- the assumption is that
|
||||
// if `dest` already exists, and we get certain errors while
|
||||
// trying to move it, we should just not bother.
|
||||
@@ -23,22 +26,29 @@ function moveFile (src, dest) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fs.link(src, dest, (err) => {
|
||||
if (err) {
|
||||
if (err.code === 'EEXIST' || err.code === 'EBUSY') {
|
||||
if (isWindows && err.code === 'EPERM') {
|
||||
// XXX This is a really weird way to handle this situation, as it
|
||||
// results in the src file being deleted even though the dest
|
||||
// might not exist. Since we pretty much always write files to
|
||||
// deterministic locations based on content hash, this is likely
|
||||
// ok (or at worst, just ends in a future cache miss). But it would
|
||||
// be worth investigating at some time in the future if this is
|
||||
// really what we want to do here.
|
||||
return resolve()
|
||||
} else if (err.code === 'EEXIST' || err.code === 'EBUSY') {
|
||||
// file already exists, so whatever
|
||||
} else if (err.code === 'EPERM' && process.platform === 'win32') {
|
||||
// file handle stayed open even past graceful-fs limits
|
||||
} else {
|
||||
return resolve()
|
||||
} else
|
||||
return reject(err)
|
||||
}
|
||||
}
|
||||
return resolve()
|
||||
} else
|
||||
return resolve()
|
||||
})
|
||||
})
|
||||
.then(() => {
|
||||
// content should never change for any reason, so make it read-only
|
||||
return Promise.all([
|
||||
unlink(src),
|
||||
process.platform !== 'win32' && chmod(dest, '0444')
|
||||
!isWindows && chmod(dest, '0444'),
|
||||
])
|
||||
})
|
||||
.catch(() => {
|
||||
@@ -49,7 +59,8 @@ function moveFile (src, dest) {
|
||||
throw err
|
||||
}
|
||||
// file doesn't already exist! let's try a rename -> copy fallback
|
||||
return move(src, dest, { Promise, fs })
|
||||
// only delete if it successfully copies
|
||||
return move(src, dest)
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
+12
-19
@@ -1,26 +1,21 @@
|
||||
'use strict'
|
||||
|
||||
const util = require('util')
|
||||
const fs = require('@npmcli/fs')
|
||||
|
||||
const figgyPudding = require('figgy-pudding')
|
||||
const fixOwner = require('./fix-owner')
|
||||
const path = require('path')
|
||||
const rimraf = util.promisify(require('rimraf'))
|
||||
const uniqueFilename = require('unique-filename')
|
||||
const { disposer } = require('./disposer')
|
||||
|
||||
const TmpOpts = figgyPudding({
|
||||
tmpPrefix: {}
|
||||
})
|
||||
|
||||
module.exports.mkdir = mktmpdir
|
||||
|
||||
function mktmpdir (cache, opts) {
|
||||
opts = TmpOpts(opts)
|
||||
const tmpTarget = uniqueFilename(path.join(cache, 'tmp'), opts.tmpPrefix)
|
||||
return fixOwner.mkdirfix(cache, tmpTarget).then(() => {
|
||||
return tmpTarget
|
||||
})
|
||||
function mktmpdir (cache, opts = {}) {
|
||||
const { tmpPrefix } = opts
|
||||
const tmpDir = path.join(cache, 'tmp')
|
||||
return fs.mkdir(tmpDir, { recursive: true, owner: 'inherit' })
|
||||
.then(() => {
|
||||
// do not use path.join(), it drops the trailing / if tmpPrefix is unset
|
||||
const target = `${tmpDir}${path.sep}${tmpPrefix || ''}`
|
||||
return fs.mkdtemp(target, { owner: 'inherit' })
|
||||
})
|
||||
}
|
||||
|
||||
module.exports.withTmp = withTmp
|
||||
@@ -28,11 +23,9 @@ module.exports.withTmp = withTmp
|
||||
function withTmp (cache, opts, cb) {
|
||||
if (!cb) {
|
||||
cb = opts
|
||||
opts = null
|
||||
opts = {}
|
||||
}
|
||||
opts = TmpOpts(opts)
|
||||
|
||||
return disposer(mktmpdir(cache, opts), rimraf, cb)
|
||||
return fs.withTempDir(path.join(cache, 'tmp'), cb, opts)
|
||||
}
|
||||
|
||||
module.exports.fix = fixtmpdir
|
||||
|
||||
+24
-28
@@ -4,9 +4,8 @@ const util = require('util')
|
||||
|
||||
const pMap = require('p-map')
|
||||
const contentPath = require('./content/path')
|
||||
const figgyPudding = require('figgy-pudding')
|
||||
const fixOwner = require('./util/fix-owner')
|
||||
const fs = require('graceful-fs')
|
||||
const fs = require('fs')
|
||||
const fsm = require('fs-minipass')
|
||||
const glob = util.promisify(require('glob'))
|
||||
const index = require('./entry-index')
|
||||
@@ -22,20 +21,16 @@ const truncate = util.promisify(fs.truncate)
|
||||
const writeFile = util.promisify(fs.writeFile)
|
||||
const readFile = util.promisify(fs.readFile)
|
||||
|
||||
const VerifyOpts = figgyPudding({
|
||||
concurrency: {
|
||||
default: 20
|
||||
},
|
||||
filter: {},
|
||||
log: {
|
||||
default: { silly () {} }
|
||||
}
|
||||
const verifyOpts = (opts) => ({
|
||||
concurrency: 20,
|
||||
log: { silly () {} },
|
||||
...opts,
|
||||
})
|
||||
|
||||
module.exports = verify
|
||||
|
||||
function verify (cache, opts) {
|
||||
opts = VerifyOpts(opts)
|
||||
opts = verifyOpts(opts)
|
||||
opts.log.silly('verify', 'verifying cache at', cache)
|
||||
|
||||
const steps = [
|
||||
@@ -45,12 +40,12 @@ function verify (cache, opts) {
|
||||
rebuildIndex,
|
||||
cleanTmp,
|
||||
writeVerifile,
|
||||
markEndTime
|
||||
markEndTime,
|
||||
]
|
||||
|
||||
return steps
|
||||
.reduce((promise, step, i) => {
|
||||
const label = step.name || `step #${i}`
|
||||
const label = step.name
|
||||
const start = new Date()
|
||||
return promise.then((stats) => {
|
||||
return step(cache, opts).then((s) => {
|
||||
@@ -59,9 +54,9 @@ function verify (cache, opts) {
|
||||
stats[k] = s[k]
|
||||
})
|
||||
const end = new Date()
|
||||
if (!stats.runTime) {
|
||||
if (!stats.runTime)
|
||||
stats.runTime = {}
|
||||
}
|
||||
|
||||
stats.runTime[label] = end - start
|
||||
return Promise.resolve(stats)
|
||||
})
|
||||
@@ -113,9 +108,9 @@ function garbageCollect (cache, opts) {
|
||||
const indexStream = index.lsStream(cache)
|
||||
const liveContent = new Set()
|
||||
indexStream.on('data', (entry) => {
|
||||
if (opts.filter && !opts.filter(entry)) {
|
||||
if (opts.filter && !opts.filter(entry))
|
||||
return
|
||||
}
|
||||
|
||||
liveContent.add(entry.integrity.toString())
|
||||
})
|
||||
return new Promise((resolve, reject) => {
|
||||
@@ -125,14 +120,14 @@ function garbageCollect (cache, opts) {
|
||||
return glob(path.join(contentDir, '**'), {
|
||||
follow: false,
|
||||
nodir: true,
|
||||
nosort: true
|
||||
nosort: true,
|
||||
}).then((files) => {
|
||||
return Promise.resolve({
|
||||
verifiedContent: 0,
|
||||
reclaimedCount: 0,
|
||||
reclaimedSize: 0,
|
||||
badContentCount: 0,
|
||||
keptSize: 0
|
||||
keptSize: 0,
|
||||
}).then((stats) =>
|
||||
pMap(
|
||||
files,
|
||||
@@ -176,14 +171,14 @@ function verifyContent (filepath, sri) {
|
||||
.then((s) => {
|
||||
const contentInfo = {
|
||||
size: s.size,
|
||||
valid: true
|
||||
valid: true,
|
||||
}
|
||||
return ssri
|
||||
.checkStream(new fsm.ReadStream(filepath), sri)
|
||||
.catch((err) => {
|
||||
if (err.code !== 'EINTEGRITY') {
|
||||
if (err.code !== 'EINTEGRITY')
|
||||
throw err
|
||||
}
|
||||
|
||||
return rimraf(filepath).then(() => {
|
||||
contentInfo.valid = false
|
||||
})
|
||||
@@ -191,9 +186,9 @@ function verifyContent (filepath, sri) {
|
||||
.then(() => contentInfo)
|
||||
})
|
||||
.catch((err) => {
|
||||
if (err.code === 'ENOENT') {
|
||||
if (err.code === 'ENOENT')
|
||||
return { size: 0, valid: false }
|
||||
}
|
||||
|
||||
throw err
|
||||
})
|
||||
}
|
||||
@@ -204,18 +199,19 @@ function rebuildIndex (cache, opts) {
|
||||
const stats = {
|
||||
missingContent: 0,
|
||||
rejectedEntries: 0,
|
||||
totalEntries: 0
|
||||
totalEntries: 0,
|
||||
}
|
||||
const buckets = {}
|
||||
for (const k in entries) {
|
||||
/* istanbul ignore else */
|
||||
if (hasOwnProperty(entries, k)) {
|
||||
const hashed = index.hashKey(k)
|
||||
const entry = entries[k]
|
||||
const excluded = opts.filter && !opts.filter(entry)
|
||||
excluded && stats.rejectedEntries++
|
||||
if (buckets[hashed] && !excluded) {
|
||||
if (buckets[hashed] && !excluded)
|
||||
buckets[hashed].push(entry)
|
||||
} else if (buckets[hashed] && excluded) {
|
||||
else if (buckets[hashed] && excluded) {
|
||||
// skip
|
||||
} else if (excluded) {
|
||||
buckets[hashed] = []
|
||||
@@ -248,7 +244,7 @@ function rebuildBucket (cache, bucket, stats, opts) {
|
||||
return index
|
||||
.insert(cache, entry.key, entry.integrity, {
|
||||
metadata: entry.metadata,
|
||||
size: entry.size
|
||||
size: entry.size,
|
||||
})
|
||||
.then(() => {
|
||||
stats.totalEntries++
|
||||
|
||||
Reference in New Issue
Block a user