From 4f74a2e01a4a6d9c873edd8e1eec0e64bff6e9ea Mon Sep 17 00:00:00 2001 From: Jordan Eldredge Date: Sun, 30 Jan 2022 14:32:55 -0500 Subject: [PATCH] Refine popular tweets --- packages/skin-database/cli.ts | 1 - packages/skin-database/package.json | 2 +- packages/skin-database/popularTweets.json | 61 +++++++-- packages/skin-database/tasks/popularTweets.ts | 6 +- yarn.lock | 129 +++++++++++------- 5 files changed, 130 insertions(+), 69 deletions(-) diff --git a/packages/skin-database/cli.ts b/packages/skin-database/cli.ts index 40abe78d..b6b9bfb4 100755 --- a/packages/skin-database/cli.ts +++ b/packages/skin-database/cli.ts @@ -24,7 +24,6 @@ import { chunk } from "./utils"; import _temp from "temp"; import Shooter from "./shooter"; import rl from "readline"; -const temp = _temp.track(); import _temp from "temp"; import Shooter from "./shooter"; diff --git a/packages/skin-database/package.json b/packages/skin-database/package.json index c30c0bba..a3df4b97 100644 --- a/packages/skin-database/package.json +++ b/packages/skin-database/package.json @@ -32,7 +32,7 @@ "node-fetch": "^2.6.0", "puppeteer": "^10.4.0", "sharp": "^0.28.0", - "sqlite3": "^4.2.0", + "sqlite3": "^5.0.2", "temp": "^0.9.0", "ts-node": "^8.10.2", "twit": "^2.2.11", diff --git a/packages/skin-database/popularTweets.json b/packages/skin-database/popularTweets.json index fc4b3d2b..cf7e0897 100644 --- a/packages/skin-database/popularTweets.json +++ b/packages/skin-database/popularTweets.json @@ -1,18 +1,53 @@ { - "25": [ - "1482170711439335426", - "1481868719114448896", - "1481838520570245122", - "1481717717656694785", - "1481506326203154435", - "1481476130905358336", - "1482472702807969792" + "50": [ + "1482472702807969792", + "1482499205675421699", + "1482563296083394561", + "1482955883315298305", + "1483137146881937412", + "1483378696480116738", + "1483559944079118337", + "1484586634645360640", + "1484496125222535171", + "1484707514847399937", + "1484737641526427651", + "1484979307009114112", + "1485009488952246274", + "1485100025688428544", + "1485371876864004099", + "1485432275433000961", + "1485734211948453889", + "1485945680837283842", + "1485824869111451650", + "1486036193598869507", + "1486519454057971712", + "1486821359636254720", + "1486881762747711489", + "1487123345124429824", + "1486247585249566720", + "1487274426966032386", + "1487334818459242497", + "1487606529884758019", + "1486579857366548480" + ], + "100": [ + "1482499205675421699", + "1482563296083394561", + "1484586634645360640", + "1485371876864004099", + "1485734211948453889", + "1486519454057971712", + "1487274426966032386", + "1487606529884758019" + ], + "150": [ + "1485371876864004099", + "1487606529884758019" + ], + "200": [ + "1485371876864004099", + "1487606529884758019" ], - "50": [], - "75": [], - "100": [], - "150": [], - "200": [], "400": [], "500": [], "600": [], diff --git a/packages/skin-database/tasks/popularTweets.ts b/packages/skin-database/tasks/popularTweets.ts index 5af71fec..6587f818 100644 --- a/packages/skin-database/tasks/popularTweets.ts +++ b/packages/skin-database/tasks/popularTweets.ts @@ -60,6 +60,7 @@ export async function popularTweets(handler: DiscordEventHandler) { const current: { [bracket: string]: string[] } = JSON.parse(currentJSON); for (const tweet of tweets) { + let notified = false; for (const [_bracket, seen] of Object.entries(current)) { const bracket = Number(_bracket); if (tweet.favorite_count > bracket && !seen.includes(tweet.id_str)) { @@ -67,7 +68,10 @@ export async function popularTweets(handler: DiscordEventHandler) { const url = tweetUrl(tweet); - await handler.handle({ type: "POPULAR_TWEET", url, bracket }); + if(!notified) { + await handler.handle({ type: "POPULAR_TWEET", url, bracket }); + notified = true + } fs.writeFileSync(JSON_FILE_NAME, JSON.stringify(current, null, 2)); } diff --git a/yarn.lock b/yarn.lock index f2f69679..fda4d6c1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3799,10 +3799,6 @@ add-dom-event-listener@^1.1.0: dependencies: object-assign "4.x" -agent-base@5: - version "5.1.1" - resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-5.1.1.tgz#e8fb3f242959db44d63be665db7a8e739537a32c" - agent-base@6: version "6.0.2" resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" @@ -4612,6 +4608,13 @@ bl@^4.0.3: inherits "^2.0.4" readable-stream "^3.4.0" +block-stream@*: + version "0.0.9" + resolved "https://registry.yarnpkg.com/block-stream/-/block-stream-0.0.9.tgz#13ebfe778a03205cfe03751481ebb4b3300c126a" + integrity sha1-E+v+d4oDIFz+A3UUgeu0szAMEmo= + dependencies: + inherits "~2.0.0" + bluebird@^3.1.5, bluebird@^3.5.1, bluebird@^3.5.5: version "3.7.2" resolved "https://registry.yarnpkg.com/bluebird/-/bluebird-3.7.2.tgz#9f229c15be272454ffa973ace0dbee79a1b0c36f" @@ -7403,16 +7406,6 @@ extract-zip@^1.6.6: mkdirp "^0.5.4" yauzl "^2.10.0" -extract-zip@^2.0.0: - version "2.0.0" - resolved "https://registry.yarnpkg.com/extract-zip/-/extract-zip-2.0.0.tgz#f53b71d44f4ff5a4527a2259ade000fb8b303492" - dependencies: - debug "^4.1.1" - get-stream "^5.1.0" - yauzl "^2.10.0" - optionalDependencies: - "@types/yauzl" "^2.9.1" - extsprintf@1.3.0: version "1.3.0" resolved "https://registry.yarnpkg.com/extsprintf/-/extsprintf-1.3.0.tgz#96918440e3041a7a414f8c52e3c574eb3c3e1e05" @@ -7916,6 +7909,16 @@ fsevents@^2.2.0, fsevents@~2.3.2: resolved "https://registry.yarnpkg.com/fsevents/-/fsevents-2.3.2.tgz#8a526f78b8fdf4623b709e0b975c52c24c02fd1a" integrity sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA== +fstream@^1.0.0, fstream@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fstream/-/fstream-1.0.12.tgz#4e8ba8ee2d48be4f7d0de505455548eae5932045" + integrity sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg== + dependencies: + graceful-fs "^4.1.2" + inherits "~2.0.0" + mkdirp ">=0.5 0" + rimraf "2" + function-bind@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/function-bind/-/function-bind-1.1.1.tgz#a56899d3ea3c9bab874bb9773b7c5ede92f4895d" @@ -8663,13 +8666,6 @@ https-proxy-agent@^2.2.1: agent-base "^4.3.0" debug "^3.1.0" -https-proxy-agent@^4.0.0: - version "4.0.0" - resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-4.0.0.tgz#702b71fb5520a132a66de1f67541d9e62154d82b" - dependencies: - agent-base "5" - debug "4" - human-signals@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-1.1.1.tgz#c5b1cd14f50aeae09ab6c59fe63ba3395fe4dfa3" @@ -8889,7 +8885,7 @@ inflight@^1.0.4: once "^1.3.0" wrappy "1" -inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.1, inherits@~2.0.3, inherits@~2.0.4: +inherits@2, inherits@2.0.4, inherits@^2.0.1, inherits@^2.0.3, inherits@^2.0.4, inherits@~2.0.0, inherits@~2.0.1, inherits@~2.0.3, inherits@~2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c" @@ -11546,7 +11542,7 @@ mkdirp-classic@^0.5.2, mkdirp-classic@^0.5.3: version "0.5.3" resolved "https://registry.yarnpkg.com/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz#fa10c9115cc6d8865be221ba47ee9bed78601113" -mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4, mkdirp@~0.5.1: +"mkdirp@>=0.5 0", mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@^0.5.3, mkdirp@^0.5.4, mkdirp@~0.5.1: version "0.5.5" resolved "https://registry.yarnpkg.com/mkdirp/-/mkdirp-0.5.5.tgz#d91cefd62d1436ca0f41620e251288d420099def" dependencies: @@ -11703,7 +11699,7 @@ node-abi@^2.21.0: dependencies: semver "^5.4.1" -node-addon-api@^3.2.0: +node-addon-api@^3.0.0, node-addon-api@^3.2.0: version "3.2.1" resolved "https://registry.yarnpkg.com/node-addon-api/-/node-addon-api-3.2.1.tgz#81325e0a2117789c0128dab65e7e38f07ceba161" integrity sha512-mmcei9JghVNDYydghQmeDX8KoAm0FAiYyIcUt/N4nhyAipB17pllZQDOJD2fotxABnt4Mdz+dKTO7eftLg4d0A== @@ -11728,6 +11724,24 @@ node-forge@0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/node-forge/-/node-forge-0.9.0.tgz#d624050edbb44874adca12bb9a52ec63cb782579" +node-gyp@3.x: + version "3.8.0" + resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-3.8.0.tgz#540304261c330e80d0d5edce253a68cb3964218c" + integrity sha512-3g8lYefrRRzvGeSowdJKAKyks8oUpLEd/DyPV4eMhVlhJ0aNaZqIrNUIPuEWWTAoPqyFkfGrM67MC69baqn6vA== + dependencies: + fstream "^1.0.0" + glob "^7.0.3" + graceful-fs "^4.1.2" + mkdirp "^0.5.0" + nopt "2 || 3" + npmlog "0 || 1 || 2 || 3 || 4" + osenv "0" + request "^2.87.0" + rimraf "2" + semver "~5.3.0" + tar "^2.0.0" + which "1" + node-gyp@^7.1.0: version "7.1.2" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-7.1.2.tgz#21a810aebb187120251c3bcec979af1587b188ae" @@ -11842,6 +11856,13 @@ node-releases@^2.0.1: resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.1.tgz#3d1d395f204f1f2f29a54358b9fb678765ad2fc5" integrity sha512-CqyzN6z7Q6aMeF/ktcMVTzhAHCEpf8SOarwpzpf8pNBY2k5/oM34UHldUwp8VKI7uxct2HxSRdJjBaZeESzcxA== +"nopt@2 || 3": + version "3.0.6" + resolved "https://registry.yarnpkg.com/nopt/-/nopt-3.0.6.tgz#c6465dbf08abcd4db359317f79ac68a646b28ff9" + integrity sha1-xkZdvwirzU2zWTF/eaxopkayj/k= + dependencies: + abbrev "1" + nopt@^4.0.1: version "4.0.3" resolved "https://registry.yarnpkg.com/nopt/-/nopt-4.0.3.tgz#a375cad9d02fd921278d954c2254d5aa57e15e48" @@ -11979,7 +12000,7 @@ npm-run-path@^4.0.0, npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" -npmlog@^4.0.1, npmlog@^4.0.2, npmlog@^4.1.2: +"npmlog@0 || 1 || 2 || 3 || 4", npmlog@^4.0.1, npmlog@^4.0.2, npmlog@^4.1.2: version "4.1.2" resolved "https://registry.yarnpkg.com/npmlog/-/npmlog-4.1.2.tgz#08a7f2a8bf734604779a9efa4ad5cc717abb954b" dependencies: @@ -12252,7 +12273,7 @@ os-tmpdir@^1.0.0, os-tmpdir@~1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/os-tmpdir/-/os-tmpdir-1.0.2.tgz#bbe67406c79aa85c5cfec766fe5734555dfa1274" -osenv@^0.1.4: +osenv@0, osenv@^0.1.4: version "0.1.5" resolved "https://registry.yarnpkg.com/osenv/-/osenv-0.1.5.tgz#85cdfafaeb28e8677f416e287592b5f3f49ea410" dependencies: @@ -13313,21 +13334,6 @@ puppeteer@^10.4.0: unbzip2-stream "1.3.3" ws "7.4.6" -puppeteer@^3.0.4: - version "3.0.4" - resolved "https://registry.yarnpkg.com/puppeteer/-/puppeteer-3.0.4.tgz#f445aae0a6732c65bbb90e963dcd6fd8fde0d780" - dependencies: - debug "^4.1.0" - extract-zip "^2.0.0" - https-proxy-agent "^4.0.0" - mime "^2.0.3" - progress "^2.0.1" - proxy-from-env "^1.0.0" - rimraf "^3.0.2" - tar-fs "^2.0.0" - unbzip2-stream "^1.3.3" - ws "^7.2.3" - q@^1.1.2: version "1.5.1" resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7" @@ -14099,6 +14105,12 @@ riff-file@^1.0.3: dependencies: byte-data "^18.0.3" +rimraf@2, rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: + version "2.7.1" + resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" + dependencies: + glob "^7.1.3" + rimraf@2.6.3, rimraf@~2.6.2: version "2.6.3" resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.6.3.tgz#b2d104fe0d8fb27cf9e0a1cda8262dd3833c6cab" @@ -14111,12 +14123,6 @@ rimraf@3.0.2, rimraf@^3.0.0, rimraf@^3.0.2: dependencies: glob "^7.1.3" -rimraf@^2.5.4, rimraf@^2.6.1, rimraf@^2.6.2, rimraf@^2.6.3: - version "2.7.1" - resolved "https://registry.yarnpkg.com/rimraf/-/rimraf-2.7.1.tgz#35797f13a7fdadc566142c29d4f07ccad483e3ec" - dependencies: - glob "^7.1.3" - ripemd160@^2.0.0, ripemd160@^2.0.1: version "2.0.2" resolved "https://registry.yarnpkg.com/ripemd160/-/ripemd160-2.0.2.tgz#a1c1a6f624751577ba5d07914cbc92850585890c" @@ -14311,6 +14317,11 @@ semver@^7.2.1, semver@^7.3.2: version "7.3.2" resolved "https://registry.yarnpkg.com/semver/-/semver-7.3.2.tgz#604962b052b81ed0786aae84389ffba70ffd3938" +semver@~5.3.0: + version "5.3.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f" + integrity sha1-myzl094C0XxgEq0yaqa00M9U+U8= + send@0.17.1: version "0.17.1" resolved "https://registry.yarnpkg.com/send/-/send-0.17.1.tgz#c1d8b059f7900f7466dd4938bdc44e11ddb376c8" @@ -14727,12 +14738,15 @@ sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" -sqlite3@^4.2.0: - version "4.2.0" - resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-4.2.0.tgz#49026d665e9fc4f922e56fb9711ba5b4c85c4901" +sqlite3@^5.0.2: + version "5.0.2" + resolved "https://registry.yarnpkg.com/sqlite3/-/sqlite3-5.0.2.tgz#00924adcc001c17686e0a6643b6cbbc2d3965083" + integrity sha512-1SdTNo+BVU211Xj1csWa8lV6KM0CtucDwRyA0VHl91wEH1Mgh7RxUpI4rVvG7OhHrzCSGaVyW5g8vKvlrk9DJA== dependencies: - nan "^2.12.1" + node-addon-api "^3.0.0" node-pre-gyp "^0.11.0" + optionalDependencies: + node-gyp "3.x" squeak@^1.0.0: version "1.3.0" @@ -15218,6 +15232,15 @@ tar-stream@^2.1.4: inherits "^2.0.3" readable-stream "^3.1.1" +tar@^2.0.0: + version "2.2.2" + resolved "https://registry.yarnpkg.com/tar/-/tar-2.2.2.tgz#0ca8848562c7299b8b446ff6a4d60cdbb23edc40" + integrity sha512-FCEhQ/4rE1zYv9rYXJw/msRqsnmlje5jHP6huWeBZ704jUTy02c5AZyWujpMR1ax6mVw9NyJMfuK2CMDWVIfgA== + dependencies: + block-stream "*" + fstream "^1.0.12" + inherits "2" + tar@^4: version "4.4.13" resolved "https://registry.yarnpkg.com/tar/-/tar-4.4.13.tgz#43b364bc52888d555298637b10d60790254ab525" @@ -15701,7 +15724,7 @@ unbzip2-stream@1.3.3: buffer "^5.2.1" through "^2.3.8" -unbzip2-stream@^1.0.9, unbzip2-stream@^1.3.3: +unbzip2-stream@^1.0.9: version "1.4.2" resolved "https://registry.yarnpkg.com/unbzip2-stream/-/unbzip2-stream-1.4.2.tgz#84eb9e783b186d8fb397515fbb656f312f1a7dbf" dependencies: @@ -16295,7 +16318,7 @@ which-module@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/which-module/-/which-module-2.0.0.tgz#d9ef07dce77b9902b8a3a8fa4b31c3e3f7e6e87a" -which@^1.2.12, which@^1.2.14, which@^1.2.9, which@^1.3.0, which@^1.3.1: +which@1, which@^1.2.12, which@^1.2.14, which@^1.2.9, which@^1.3.0, which@^1.3.1: version "1.3.1" resolved "https://registry.yarnpkg.com/which/-/which-1.3.1.tgz#a45043d54f5805316da8d62f9f50918d3da70b0a" dependencies: