From fc3c641e52e55af2f015023c53371a85a039c847 Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 23 Jun 2019 16:23:09 -0400 Subject: [PATCH] Improve logging --- experiments/skin-database/cli.js | 5 +- .../discord-bot/commands/archive.js | 32 ++-- .../skin-database/discord-bot/index.js | 15 +- experiments/skin-database/discord-bot/s3.js | 3 - .../skin-database/discord-bot/utils.js | 21 ++- experiments/skin-database/logger.js | 18 +++ experiments/skin-database/package.json | 1 + experiments/skin-database/yarn.lock | 152 +++++++++++++++++- 8 files changed, 216 insertions(+), 31 deletions(-) create mode 100644 experiments/skin-database/logger.js diff --git a/experiments/skin-database/cli.js b/experiments/skin-database/cli.js index 4182cb20..2371c99a 100755 --- a/experiments/skin-database/cli.js +++ b/experiments/skin-database/cli.js @@ -3,6 +3,7 @@ const argv = require("yargs").argv; const findTweetableSkin = require("./tasks/findTweetableSkins"); const fetchInternetArchiveMetadata = require("./tasks/fetchInternetArchiveMetadata"); const path = require("path"); +const logger = require("./logger"); const Skins = require("./data/skins"); const { spawn } = require("child_process"); @@ -46,9 +47,7 @@ async function main() { //, "--dry" ] ); - console.log({ output }); - - console.log("Done"); + logger.info("Tweeted a skin", { md5, filename, ...output }); break; case "fetch-metadata": console.log("Going to download metadata from the Internet Archive"); diff --git a/experiments/skin-database/discord-bot/commands/archive.js b/experiments/skin-database/discord-bot/commands/archive.js index 72ea55c7..80a0b450 100644 --- a/experiments/skin-database/discord-bot/commands/archive.js +++ b/experiments/skin-database/discord-bot/commands/archive.js @@ -3,9 +3,10 @@ const path = require("path"); const fetch = require("node-fetch"); const md5Buffer = require("md5"); const config = require("../../config"); +const Skins = require("../../data/skins"); +const Utils = require("../utils"); const { getInfo } = require("../info"); -const Skin = require("./skin"); async function handler(message) { const { attachments } = message; @@ -26,9 +27,27 @@ async function handler(message) { }) ); + // Compute MD5 + // Upload to S3 + // Record the filepath + // Take a screenshot + // Upload screenshot to S3 + // Extract the readme + // Extract the emails + // Extract the average color + // Upload to Internet Archive + // Store the Internet Archive item name + // Construct IA Webamp URL + for (const file of files) { - const info = await getInfo(file.md5); - if (info == null) { + const skin = await Skins.getSkinByMd5(file.md5); + if (skin != null) { + await message.channel.send(`This skin is already in our collection.`); + await Utils.postSkin({ + md5: file.md5, + dest: message.channel + }); + } else { fs.writeFileSync( path.join( config.uploadDir, @@ -40,14 +59,9 @@ async function handler(message) { ); await message.channel.send( `Thanks! ${ - file.filename + file.filename } is a brand new skin. 👏 It has been queued for archiving.` ); - } else { - await message.channel.send( - `We already have ${file.filename} in our collection` - ); - await Skin.handler(message, [file.md5]); } } } diff --git a/experiments/skin-database/discord-bot/index.js b/experiments/skin-database/discord-bot/index.js index 77cf43b1..021419bf 100644 --- a/experiments/skin-database/discord-bot/index.js +++ b/experiments/skin-database/discord-bot/index.js @@ -2,6 +2,7 @@ const fs = require("fs"); const path = require("path"); const Discord = require("discord.js"); const config = require("../config"); +const logger = require("../logger"); const client = new Discord.Client(); @@ -52,17 +53,17 @@ client.on("message", async message => { return; } const command = rawCommand.slice(1); - console.log("New command: ", command); - console.log("User: ", message.author.username); - console.log("Arguments: ", args); + logger.info('User triggered WebampBot command', {command, user: message.author.username, args, channel: message.channel.name || 'DM'}); const handler = handlers[command]; - if (handler != null) { - handler(message, args); + if(handler == null) { + logger.warn('Unknown command', {command, user: message.author.username, args}); + return; } + handler(message, args); }); -client.on("error", error => { - console.error("The WebSocket encountered an error:", error); +client.on("error", e => { + logger.error('The WebSocket encountered an error:', e); }); client.login(config.discordToken); diff --git a/experiments/skin-database/discord-bot/s3.js b/experiments/skin-database/discord-bot/s3.js index b4d9f54c..c497d575 100644 --- a/experiments/skin-database/discord-bot/s3.js +++ b/experiments/skin-database/discord-bot/s3.js @@ -69,14 +69,12 @@ async function getStats() { } async function getSkinToReview() { - console.log("Reading from s3..."); const [filenames, approved, rejected, tweeted] = await Promise.all([ getFile("filenames.txt"), getFile("approved.txt"), getFile("rejected.txt"), getFile("tweeted.txt") ]); - console.log("Got all files"); const approvedSet = new Set(getLines(approved)); const rejectedSet = new Set(getLines(rejected)); @@ -94,7 +92,6 @@ async function getSkinToReview() { tweetedSet.has(md5) ); }); - console.log(toReview.length, "skins to review"); return toReview[0]; } diff --git a/experiments/skin-database/discord-bot/utils.js b/experiments/skin-database/discord-bot/utils.js index eb4ac3c3..4b5b7671 100644 --- a/experiments/skin-database/discord-bot/utils.js +++ b/experiments/skin-database/discord-bot/utils.js @@ -2,6 +2,8 @@ const Discord = require("discord.js"); const rgbHex = require("rgb-hex"); const Skins = require("../data/skins"); const { approve, reject } = require("./s3"); +const logger = require("../logger"); +const { approve, reject } = require("./s3"); const filter = reaction => { return ["👍", "👎"].some(name => reaction.emoji.name === name); @@ -9,6 +11,10 @@ const filter = reaction => { async function postSkin({ md5, title, dest }) { const skin = await Skins.getSkinByMd5(md5); + if (skin == null) { + logger.warn("Could not find skin for md5", { md5 }); + return; + } const { canonicalFilename, screenshotUrl, @@ -23,7 +29,6 @@ async function postSkin({ md5, title, dest }) { internetArchiveItemName, readmeText } = skin; - console.log(skin); title = title ? title(canonicalFilename) : canonicalFilename; const embed = new Discord.RichEmbed() @@ -43,11 +48,15 @@ async function postSkin({ md5, title, dest }) { if (String(color).length === 6) { embed.setColor(`#${color}`); } else { - console.log("Did not get a safe color from ", averageColor); - console.log("Got ", color); + logger.warn( + "Did not get a safe color from ", + averageColor, + "got", + color + ); } } catch (e) { - console.error("could not use color", averageColor); + logger.error("Could not use color", averageColor); } } if (emails != null && emails.length) { @@ -81,7 +90,7 @@ async function postSkin({ md5, title, dest }) { switch (vote.emoji.name) { case "👍": await approve(md5); - console.log(`${user.username} approved ${md5}`); + logger.info(`${user.username} approved ${md5}`); await msg.channel.send( `${canonicalFilename} was approved by ${user.username}` ); @@ -89,7 +98,7 @@ async function postSkin({ md5, title, dest }) { break; case "👎": await reject(md5); - console.log(`${user.username} rejected ${md5}`); + logger.info(`${user.username} rejected ${md5}`); await msg.channel.send( `${canonicalFilename} was rejected by ${user.username}` ); diff --git a/experiments/skin-database/logger.js b/experiments/skin-database/logger.js new file mode 100644 index 00000000..02a640df --- /dev/null +++ b/experiments/skin-database/logger.js @@ -0,0 +1,18 @@ +const { createLogger, format, transports } = require('winston'); + +const logger = createLogger({ + level: 'info', + format: format.combine( + format.timestamp({ + format: 'YYYY-MM-DD HH:mm:ss' + }), + format.errors({ stack: true }), + format.splat(), + format.json() + ), + transports: [ + new transports.File({ filename: 'combined.log' }) + ] +}); + +module.exports = logger; diff --git a/experiments/skin-database/package.json b/experiments/skin-database/package.json index 91a0dff3..8236b619 100644 --- a/experiments/skin-database/package.json +++ b/experiments/skin-database/package.json @@ -15,6 +15,7 @@ "puppeteer": "^1.17.0", "rgb-hex": "^2.1.0", "temp": "^0.9.0", + "winston": "^3.2.1", "yargs": "^13.2.4" }, "scripts": { diff --git a/experiments/skin-database/yarn.lock b/experiments/skin-database/yarn.lock index a6989f13..54538752 100644 --- a/experiments/skin-database/yarn.lock +++ b/experiments/skin-database/yarn.lock @@ -108,6 +108,12 @@ async-limiter@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/async-limiter/-/async-limiter-1.0.0.tgz#78faed8c3d074ab81f22b4e985d79e8738f720f8" +async@^2.6.1: + version "2.6.2" + resolved "https://registry.yarnpkg.com/async/-/async-2.6.2.tgz#18330ea7e6e313887f5d2f2a904bac6fe4dd5381" + dependencies: + lodash "^4.17.11" + atob@^2.1.1: version "2.1.2" resolved "https://registry.yarnpkg.com/atob/-/atob-2.1.2.tgz#6d9517eb9e030d2436666651e86bd9f6f13533c9" @@ -377,7 +383,7 @@ collection-visit@^1.0.0: map-visit "^1.0.0" object-visit "^1.0.0" -color-convert@^1.9.0: +color-convert@^1.9.0, color-convert@^1.9.1: version "1.9.3" resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8" dependencies: @@ -387,6 +393,39 @@ color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" +color-name@^1.0.0: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + +color-string@^1.5.2: + version "1.5.3" + resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.3.tgz#c9bbc5f01b58b5492f3d6857459cb6590ce204cc" + dependencies: + color-name "^1.0.0" + simple-swizzle "^0.2.2" + +color@3.0.x: + version "3.0.0" + resolved "https://registry.yarnpkg.com/color/-/color-3.0.0.tgz#d920b4328d534a3ac8295d68f7bd4ba6c427be9a" + dependencies: + color-convert "^1.9.1" + color-string "^1.5.2" + +colornames@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/colornames/-/colornames-1.1.1.tgz#f8889030685c7c4ff9e2a559f5077eb76a816f96" + +colors@^1.2.1: + version "1.3.3" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d" + +colorspace@1.1.x: + version "1.1.2" + resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5" + dependencies: + color "3.0.x" + text-hex "1.0.x" + commander@~2.8.1: version "2.8.1" resolved "https://registry.yarnpkg.com/commander/-/commander-2.8.1.tgz#06be367febfda0c330aa1e2a072d3dc9762425d4" @@ -582,6 +621,14 @@ destroy@~1.0.4: version "1.0.4" resolved "https://registry.yarnpkg.com/destroy/-/destroy-1.0.4.tgz#978857442c44749e4206613e37946205826abd80" +diagnostics@^1.1.1: + version "1.1.1" + resolved "https://registry.yarnpkg.com/diagnostics/-/diagnostics-1.1.1.tgz#cab6ac33df70c9d9a727490ae43ac995a769b22a" + dependencies: + colorspace "1.1.x" + enabled "1.0.x" + kuler "1.0.x" + dir-glob@2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-2.0.0.tgz#0b205d2b6aef98238ca286598a8204d29d0a0034" @@ -644,6 +691,12 @@ emoji-regex@^7.0.1: version "7.0.3" resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" +enabled@1.0.x: + version "1.0.2" + resolved "https://registry.yarnpkg.com/enabled/-/enabled-1.0.2.tgz#965f6513d2c2d1c5f4652b64a2e3396467fc2f93" + dependencies: + env-variable "0.0.x" + encodeurl@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/encodeurl/-/encodeurl-1.0.2.tgz#ad3ff4c86ec2d029322f5a02c3a9a606c95b3f59" @@ -654,6 +707,10 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" +env-variable@0.0.x: + version "0.0.5" + resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88" + error-ex@^1.2.0: version "1.3.2" resolved "https://registry.yarnpkg.com/error-ex/-/error-ex-1.3.2.tgz#b4ac40648107fdcdcfae242f428bea8a14d4f1bf" @@ -832,6 +889,10 @@ fast-glob@^2.0.2: merge2 "^1.2.3" micromatch "^3.1.10" +fast-safe-stringify@^2.0.4: + version "2.0.6" + resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz#04b26106cc56681f51a044cfc0d76cf0008ac2c2" + fd-slicer@~1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/fd-slicer/-/fd-slicer-1.0.1.tgz#8b5bcbd9ec327c5041bf9ab023fd6750f1177e65" @@ -844,6 +905,10 @@ fd-slicer@~1.1.0: dependencies: pend "~1.2.0" +fecha@^2.3.3: + version "2.3.3" + resolved "https://registry.yarnpkg.com/fecha/-/fecha-2.3.3.tgz#948e74157df1a32fd1b12c3a3c3cdcb6ec9d96cd" + figures@^1.3.5: version "1.7.0" resolved "https://registry.yarnpkg.com/figures/-/figures-1.7.0.tgz#cbe1e3affcf1cd44b80cadfed28dc793a9701d2e" @@ -1240,6 +1305,10 @@ is-arrayish@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.2.1.tgz#77c99840527aa8ecb1a8ba697b80645a7a926a9d" +is-arrayish@^0.3.1: + version "0.3.2" + resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03" + is-buffer@^1.1.5, is-buffer@~1.1.1: version "1.1.6" resolved "https://registry.yarnpkg.com/is-buffer/-/is-buffer-1.1.6.tgz#efaa2ea9daa0d7ab2ea13a97b2b8ad51fefbe8be" @@ -1411,6 +1480,12 @@ kind-of@^6.0.0, kind-of@^6.0.2: version "6.0.2" resolved "https://registry.yarnpkg.com/kind-of/-/kind-of-6.0.2.tgz#01146b36a6218e64e58f3a8d66de5d7fc6f6d051" +kuler@1.0.x: + version "1.0.1" + resolved "https://registry.yarnpkg.com/kuler/-/kuler-1.0.1.tgz#ef7c784f36c9fb6e16dd3150d152677b2b0228a6" + dependencies: + colornames "^1.1.1" + lcid@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/lcid/-/lcid-2.0.0.tgz#6ef5d2df60e52f82eb228a4c373e8d1f397253cf" @@ -1434,6 +1509,10 @@ locate-path@^3.0.0: p-locate "^3.0.0" path-exists "^3.0.0" +lodash@^4.17.11: + version "4.17.11" + resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.11.tgz#b39ea6229ef607ecd89e2c8df12536891cac9b8d" + logalot@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/logalot/-/logalot-2.1.0.tgz#5f8e8c90d304edf12530951a5554abb8c5e3f552" @@ -1441,6 +1520,16 @@ logalot@^2.0.0: figures "^1.3.5" squeak "^1.0.0" +logform@^2.1.1: + version "2.1.2" + resolved "https://registry.yarnpkg.com/logform/-/logform-2.1.2.tgz#957155ebeb67a13164069825ce67ddb5bb2dd360" + dependencies: + colors "^1.2.1" + fast-safe-stringify "^2.0.4" + fecha "^2.3.3" + ms "^2.1.1" + triple-beam "^1.3.0" + long@^4.0.0: version "4.0.0" resolved "https://registry.yarnpkg.com/long/-/long-4.0.0.tgz#9a7b71cfb7d361a194ea555241c92f7468d5bf28" @@ -1789,6 +1878,10 @@ once@^1.3.0, once@^1.3.1, once@^1.4.0: dependencies: wrappy "1" +one-time@0.0.4: + version "0.0.4" + resolved "https://registry.yarnpkg.com/one-time/-/one-time-0.0.4.tgz#f8cdf77884826fe4dff93e3a9cc37b1e4480742e" + optipng-bin@^6.0.0: version "6.0.0" resolved "https://registry.yarnpkg.com/optipng-bin/-/optipng-bin-6.0.0.tgz#376120fa79d5e71eee2f524176efdd3a5eabd316" @@ -2084,7 +2177,7 @@ read-pkg@^1.0.0: normalize-package-data "^2.3.2" path-type "^1.0.0" -readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5: +readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@^2.3.0, readable-stream@^2.3.5, readable-stream@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf" dependencies: @@ -2096,6 +2189,14 @@ readable-stream@^2.0.0, readable-stream@^2.2.2, readable-stream@^2.3.0, readable string_decoder "~1.1.1" util-deprecate "~1.0.1" +readable-stream@^3.1.1: + version "3.4.0" + resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc" + dependencies: + inherits "^2.0.3" + string_decoder "^1.1.1" + util-deprecate "^1.0.1" + redent@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/redent/-/redent-1.0.0.tgz#cf916ab1fd5f1f16dfb20822dd6ec7f730c2afde" @@ -2292,6 +2393,12 @@ signal-exit@^3.0.0: version "3.0.2" resolved "https://registry.yarnpkg.com/signal-exit/-/signal-exit-3.0.2.tgz#b5fdc08f1287ea1178628e415e25132b73646c6d" +simple-swizzle@^0.2.2: + version "0.2.2" + resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a" + dependencies: + is-arrayish "^0.3.1" + slash@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" @@ -2405,6 +2512,10 @@ squeak@^1.0.0: console-stream "^0.1.1" lpad-align "^1.0.1" +stack-trace@0.0.x: + version "0.0.10" + resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0" + static-extend@^0.1.1: version "0.1.2" resolved "https://registry.yarnpkg.com/static-extend/-/static-extend-0.1.2.tgz#60809c39cbff55337226fd5e0b520f341f1fb5c6" @@ -2428,6 +2539,12 @@ string-width@^3.0.0, string-width@^3.1.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" +string_decoder@^1.1.1: + version "1.2.0" + resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.2.0.tgz#fe86e738b19544afe70469243b2a1ee9240eae8d" + dependencies: + safe-buffer "~5.1.0" + string_decoder@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8" @@ -2507,6 +2624,10 @@ tempfile@^2.0.0: temp-dir "^1.0.0" uuid "^3.0.1" +text-hex@1.0.x: + version "1.0.0" + resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5" + through@^2.3.8: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" @@ -2555,6 +2676,10 @@ trim-repeated@^1.0.0: dependencies: escape-string-regexp "^1.0.2" +triple-beam@^1.2.0, triple-beam@^1.3.0: + version "1.3.0" + resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9" + tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" @@ -2634,7 +2759,7 @@ use@^3.1.0: version "3.1.1" resolved "https://registry.yarnpkg.com/use/-/use-3.1.1.tgz#d50c8cac79a19fbc20f2911f56eb973f4e10070f" -util-deprecate@~1.0.1: +util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" @@ -2667,6 +2792,27 @@ which@^1.2.9: dependencies: isexe "^2.0.0" +winston-transport@^4.3.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.3.0.tgz#df68c0c202482c448d9b47313c07304c2d7c2c66" + dependencies: + readable-stream "^2.3.6" + triple-beam "^1.2.0" + +winston@^3.2.1: + version "3.2.1" + resolved "https://registry.yarnpkg.com/winston/-/winston-3.2.1.tgz#63061377976c73584028be2490a1846055f77f07" + dependencies: + async "^2.6.1" + diagnostics "^1.1.1" + is-stream "^1.1.0" + logform "^2.1.1" + one-time "0.0.4" + readable-stream "^3.1.1" + stack-trace "0.0.x" + triple-beam "^1.3.0" + winston-transport "^4.3.0" + wrap-ansi@^5.1.0: version "5.1.0" resolved "https://registry.yarnpkg.com/wrap-ansi/-/wrap-ansi-5.1.0.tgz#1fd1f67235d5b6d0fee781056001bfb694c03b09"