CasperSecurity
const os = require('os')
const path = require('path')
const { format, promisify } = require('util')
const rimraf = promisify(require('rimraf'))
const glob = promisify(require('glob'))
const MiniPass = require('minipass')
const fsMiniPass = require('fs-minipass')
const log = require('./log-shim')
const withChownSync = require('./with-chown-sync')
const padZero = (n, length) => n.toString().padStart(length.toString().length, '0')
const _logHandler = Symbol('logHandler')
const _formatLogItem = Symbol('formatLogItem')
const _getLogFilePath = Symbol('getLogFilePath')
const _openLogFile = Symbol('openLogFile')
const _cleanLogs = Symbol('cleanlogs')
const _endStream = Symbol('endStream')
const _isBuffered = Symbol('isBuffered')
class LogFiles {
// If we write multiple log files we want them all to have the same
// identifier for sorting and matching purposes
#logId = null
// Default to a plain minipass stream so we can buffer
// initial writes before we know the cache location
#logStream = null
// We cap log files at a certain number of log events per file.
// Note that each log event can write more than one line to the
// file. Then we rotate log files once this number of events is reached
#MAX_LOGS_PER_FILE = null
// Now that we write logs continuously we need to have a backstop
// here for infinite loops that still log. This is also partially handled
// by the config.get('max-files') option, but this is a failsafe to
// prevent runaway log file creation
#MAX_FILES_PER_PROCESS = null
#fileLogCount = 0
#totalLogCount = 0
#dir = null
#logsMax = null
#files = []
constructor ({
maxLogsPerFile = 50_000,
maxFilesPerProcess = 5,
} = {}) {
this.#logId = LogFiles.logId(new Date())
this.#MAX_LOGS_PER_FILE = maxLogsPerFile
this.#MAX_FILES_PER_PROCESS = maxFilesPerProcess
this.on()
}
static logId (d) {
return d.toISOString().replace(/[.:]/g, '_')
}
static format (count, level, title, ...args) {
let prefix = `${count} ${level}`
if (title) {
prefix += ` ${title}`
}
return format(...args)
.split(/\r?\n/)
.reduce((lines, line) =>
lines += prefix + (line ? ' ' : '') + line + os.EOL,
''
)
}
on () {
this.#logStream = new MiniPass()
process.on('log', this[_logHandler])
}
off () {
process.off('log', this[_logHandler])
this[_endStream]()
}
load ({ dir, logsMax } = {}) {
this.#dir = dir
this.#logsMax = logsMax
// Log stream has already ended
if (!this.#logStream) {
return
}
// Pipe our initial stream to our new file stream and
// set that as the new log logstream for future writes
const initialFile = this[_openLogFile]()
if (initialFile) {
this.#logStream = this.#logStream.pipe(initialFile)
}
// Kickoff cleaning process. This is async but it wont delete
// our next log file since it deletes oldest first. Return the
// result so it can be awaited in tests
return this[_cleanLogs]()
}
log (...args) {
this[_logHandler](...args)
}
get files () {
return this.#files
}
get [_isBuffered] () {
return this.#logStream instanceof MiniPass
}
[_endStream] (output) {
if (this.#logStream) {
this.#logStream.end(output)
this.#logStream = null
}
}
[_logHandler] = (level, ...args) => {
// Ignore pause and resume events since we
// write everything to the log file
if (level === 'pause' || level === 'resume') {
return
}
// If the stream is ended then do nothing
if (!this.#logStream) {
return
}
const logOutput = this[_formatLogItem](level, ...args)
if (this[_isBuffered]) {
// Cant do anything but buffer the output if we dont
// have a file stream yet
this.#logStream.write(logOutput)
return
}
// Open a new log file if we've written too many logs to this one
if (this.#fileLogCount >= this.#MAX_LOGS_PER_FILE) {
// Write last chunk to the file and close it
this[_endStream](logOutput)
if (this.#files.length >= this.#MAX_FILES_PER_PROCESS) {
// but if its way too many then we just stop listening
this.off()
} else {
// otherwise we are ready for a new file for the next event
this.#logStream = this[_openLogFile]()
}
} else {
this.#logStream.write(logOutput)
}
}
[_formatLogItem] (...args) {
this.#fileLogCount += 1
return LogFiles.format(this.#totalLogCount++, ...args)
}
[_getLogFilePath] (prefix, suffix, sep = '-') {
return path.resolve(this.#dir, prefix + sep + 'debug' + sep + suffix + '.log')
}
[_openLogFile] () {
// Count in filename will be 0 indexed
const count = this.#files.length
try {
const logStream = withChownSync(
// Pad with zeros so that our log files are always sorted properly
// We never want to write files ending in `-9.log` and `-10.log` because
// log file cleaning is done by deleting the oldest so in this example
// `-10.log` would be deleted next
this[_getLogFilePath](this.#logId, padZero(count, this.#MAX_FILES_PER_PROCESS)),
// Some effort was made to make the async, but we need to write logs
// during process.on('exit') which has to be synchronous. So in order
// to never drop log messages, it is easiest to make it sync all the time
// and this was measured to be about 1.5% slower for 40k lines of output
(f) => new fsMiniPass.WriteStreamSync(f, { flags: 'a' })
)
if (count > 0) {
// Reset file log count if we are opening
// after our first file
this.#fileLogCount = 0
}
this.#files.push(logStream.path)
return logStream
} catch (e) {
// XXX: do something here for errors?
// log to display only?
return null
}
}
async [_cleanLogs] () {
// module to clean out the old log files
// this is a best-effort attempt. if a rm fails, we just
// log a message about it and move on. We do return a
// Promise that succeeds when we've tried to delete everything,
// just for the benefit of testing this function properly.
if (typeof this.#logsMax !== 'number') {
return
}
try {
// Handle the old (prior to 8.2.0) log file names which did not have an counter suffix
// so match by anything after `-debug` and before `.log` (including nothing)
const logGlob = this[_getLogFilePath]('*-', '*', '')
// Always ignore the currently written files
const files = await glob(logGlob, { ignore: this.#files })
const toDelete = files.length - this.#logsMax
if (toDelete <= 0) {
return
}
log.silly('logfile', `start cleaning logs, removing ${toDelete} files`)
for (const file of files.slice(0, toDelete)) {
try {
await rimraf(file)
} catch (e) {
log.silly('logfile', 'error removing log file', file, e)
}
}
} catch (e) {
log.warn('logfile', 'error cleaning log files', e)
}
}
}
module.exports = LogFiles