diff --git a/nodes.md b/nodes.md new file mode 100644 index 00000000..a969dee3 --- /dev/null +++ b/nodes.md @@ -0,0 +1,49 @@ +# States + +1. ~Lazy loading initial results~ +2. Above the fold placeholders +3. Above the fold images +4. Scrolling existing placeholders +5. Scrolling unfetched placeholders + Show random colors +6. Clicked scrollbar and went to the bottom of the page +7. Cliked an image + +We should include the first ~100 results in the initial payload + +Loading a set of search results + +1. Include teh color and md5 in the results + +Each segment should have a state: + +- Loaded +- Loading +- Errored +- Not Requested + +When we want to show a placeholder we: + +- Check if we already have it +- If so, we show it +- If not, pick a random color, and then (asycn) we check which segment it would be in +- If that segement is loading, noop +- If the segment has not been requested, we request it and mark it as loading +- When a segment comes back, we update our state + +When a user clicks a skin + +- If the skin is loaded (we have the md5) we take the user to the permalink +- If the skin is not loaded, we check which segment it would be in +- If the segment has not yet been requested (unlikely, since it should have been requested when it came into view) we request it +- If the segment is loading, we await its completion. +- If 500ms passes before the load completes, we show a spinnner or something +- When the loading completes we take the user to the permalink + +## Functions we'll need + +- `getSegmentForIndex(i)` +- `getMd5ForIndex(i)` (syncronous, but may kick off an async action) +- `getColorForIndex(i)` (syncronus. May give a BS result if it does not have one right away, may kick off an async action) +- `getSegmentStatus(segmentId)` +- `awaitMd5ForIndex(i)` diff --git a/scripts/createSearchIndex.js b/scripts/createSearchIndex.js deleted file mode 100644 index 37e95e96..00000000 --- a/scripts/createSearchIndex.js +++ /dev/null @@ -1,71 +0,0 @@ -const path = require("path"); -const fs = require("fs"); -const algoliasearch = require("algoliasearch"); - -const client = algoliasearch("HQ9I5Z6IM5", "f5357f4070cdb6ed652d9c3feeede89f"); -const index = client.initIndex("Skins"); - -const CACHE_PATH = "/Volumes/Mobile Backup/skins/cache/"; -const info = JSON.parse( - fs.readFileSync(path.join(CACHE_PATH, "info.json"), "utf8") -); - -function tuncate(str, len) { - const overflow = str.length - len; - if (overflow < 0) { - return str; - } - - const half = Math.floor((len - 1) / 2); - - const start = str.slice(0, half); - const end = str.slice(-half); - return `${start} ########### ${end}`; -} - -async function buildSkinIndex(skin) { - const { md5, filePaths } = skin; - if (!filePaths || filePaths.length === 0) { - console.warn("no file name for ", md5); - return; - } - const fileName = path.basename(filePaths[0]); - let readmeText = null; - if (skin.readmePath) { - readmeText = tuncate(fs.readFileSync(skin.readmePath, "utf8"), 4800); - } - return { - objectID: skin.md5, - //md5, - //fileName, - // emails: skin.emails || null, - // readmeText - // color: skin.averageColor - // twitterLikes: Number(skin.twitterLikes || 0) - }; -} - -const indexesPromise = Promise.all( - Object.values(info) - .filter((skin) => skin.type === "CLASSIC") - .map((skin) => buildSkinIndex(skin)) -); - -async function go({ dry = true }) { - const indexes = await indexesPromise; - - if (dry) { - console.log("Index turned off. Turn it on if you really mean it"); - return; - } - console.log("Writing index"); - const results = await new Promise((resolve, reject) => { - index.partialUpdateObjects(indexes, function (err, content) { - if (err != null) reject(err); - resolve(content); - }); - }); - console.log("done!", results); -} - -go({ dry: false }); // .then(content => console.log("Updated index for:", content.length)); diff --git a/scripts/findSkins.js b/scripts/findSkins.js deleted file mode 100644 index 56c4e5a6..00000000 --- a/scripts/findSkins.js +++ /dev/null @@ -1,77 +0,0 @@ -const fs = require("fs"); -const path = require("path"); -const Bluebird = require("bluebird"); - -const info = JSON.parse( - fs.readFileSync("/Volumes/Mobile Backup/skins/cache/info.json", "utf8") -); - -const getFileData = async () => { - const fileData = await Bluebird.map( - Object.values(info).filter(skin => skin.type === "CLASSIC"), - async skin => { - const md5 = skin.md5; - const filePaths = skin.filePaths; - if (!filePaths || filePaths.length === 0) { - console.warn("no file name for ", md5); - return; - } - if (skin.twitterLikes) { - // console.log({ skin }); - } - const fileName = path.basename(filePaths[0]); - return { - color: skin.averageColor, - favorites: skin.twitterLikes || 0, - fileName, - md5 - }; - }, - { concurrency: 10 } - ); - const orderedFileData = fileData.sort((a, b) => { - return b.favorites - a.favorites; - }); - return orderedFileData.map(({ favorites, ...rest }) => rest); -}; - -const SKIN_DATA_ROOT = "public/skinData"; -const BOOTSTRAP_SKIN_DATA_ROOT = "src"; -const CHUNK_SIZE = 500; -getFileData().then(data => { - let start = 0; - const chunkFileNames = []; - let chunkNumber = 0; - while (start <= data.length) { - const end = start + CHUNK_SIZE; - - const chunk = data.slice(start, end); - const json = JSON.stringify(chunk); - const fileName = `skins-${chunkNumber}.json`; - fs.writeFileSync(path.join(SKIN_DATA_ROOT, fileName), json, "utf8"); - if (chunkNumber === 0) { - fs.writeFileSync( - path.join(BOOTSTRAP_SKIN_DATA_ROOT, fileName), - json, - "utf8" - ); - } - chunkFileNames.push(fileName); - start = end; - chunkNumber++; - } - const json = JSON.stringify( - { - chunkSize: CHUNK_SIZE, - numberOfSkins: data.length, - chunkFileNames - }, - null, - 2 - ); - fs.writeFileSync( - path.join(BOOTSTRAP_SKIN_DATA_ROOT, "skins.json"), - json, - "utf8" - ); -}); diff --git a/scripts/utils.js b/scripts/utils.js deleted file mode 100644 index 40a5ac02..00000000 --- a/scripts/utils.js +++ /dev/null @@ -1,63 +0,0 @@ -const path = require("path"); -const fs = require("fs"); -const md5File = require("md5-file"); -var { exec } = require("child_process"); - -const METADATA_ROOT = path.join(__dirname, "../metadata/"); - -function getSkinMetadataPath(md5, key) { - return path.join(METADATA_ROOT, md5, `${key}.json`); -} - -function getSkinMetadata(md5, key) { - const skinMetadataPath = getSkinMetadataPath(md5, key); - return new Promise((resolve, reject) => { - fs.readFile(skinMetadataPath, "utf8", (err, data) => { - if (err != null) reject(err); - resolve(JSON.parse(data)); - }); - }); -} - -function ensureDirectoryExistence(filePath) { - var dirname = path.dirname(filePath); - if (fs.existsSync(dirname)) { - return true; - } - ensureDirectoryExistence(dirname); - fs.mkdirSync(dirname); -} - -async function writeSkinMetadata(md5, key, data) { - const skinMetadataPath = getSkinMetadataPath(md5, key); - ensureDirectoryExistence(skinMetadataPath); - return new Promise((resolve, reject) => { - fs.writeFile( - skinMetadataPath, - JSON.stringify(data, null, 2), - "utf8", - err => { - if (err != null) { - reject(err); - } - resolve(skinMetadataPath); - } - ); - }); -} - -function getFileMd5(filePath) { - return new Promise((resolve, reject) => { - md5File(filePath, (err, hash) => { - if (err) reject(err); - - resolve(hash); - }); - }); -} - -module.exports = { - getSkinMetadata, - writeSkinMetadata, - getFileMd5 -}; diff --git a/src/App.js b/src/App.js index dd74fda1..24c7faf2 100644 --- a/src/App.js +++ b/src/App.js @@ -9,12 +9,12 @@ import FocusedSkin from "./FocusedSkin"; import * as Selectors from "./redux/selectors"; import { ABOUT_PAGE } from "./constants"; import * as Utils from "./utils"; -import { SKIN_WIDTH, SKIN_RATIO } from "./constants"; +import { SCREENSHOT_WIDTH, SKIN_RATIO } from "./constants"; // Render your table -const getTableDimensions = (windowWidth) => { - const columnCount = Math.floor(windowWidth / SKIN_WIDTH); +const getTableDimensions = (windowWidth, scale) => { + const columnCount = Math.floor(windowWidth / (SCREENSHOT_WIDTH * scale)); const columnWidth = windowWidth / columnCount; // TODO: Consider flooring this to get things aligned to the pixel const rowHeight = columnWidth * SKIN_RATIO; return { columnWidth, rowHeight, columnCount }; @@ -36,7 +36,8 @@ function useWindowSize() { function App(props) { const { windowWidth, windowHeight } = useWindowSize(); const { columnWidth, rowHeight, columnCount } = getTableDimensions( - windowWidth + windowWidth, + props.scale ); return (
@@ -73,6 +74,7 @@ const mapStateToProps = (state) => ({ selectedSkinHash: Selectors.getSelectedSkinHash(state), overlayShouldAnimate: Selectors.overlayShouldAnimate(state), aboutPage: Selectors.getActiveContentPage(state) === ABOUT_PAGE, + scale: state.scale, }); export default connect(mapStateToProps)(App); diff --git a/src/App.test.js b/src/App.test.js deleted file mode 100644 index 23c181f0..00000000 --- a/src/App.test.js +++ /dev/null @@ -1,9 +0,0 @@ -import React from "react"; -import ReactDOM from "react-dom"; -import App from "./App"; - -it("renders without crashing", () => { - const div = document.createElement("div"); - ReactDOM.render(, div); - ReactDOM.unmountComponentAtNode(div); -}); diff --git a/src/Header.js b/src/Header.js index a73e9536..d81b58fb 100644 --- a/src/Header.js +++ b/src/Header.js @@ -14,7 +14,7 @@ class Header extends React.Component { } componentDidMount() { - const handler = e => { + const handler = (e) => { // slash if (e.keyCode === 191) { if (this._inputRef == null) { @@ -41,7 +41,7 @@ class Header extends React.Component {

{ + onClick={(e) => { if (Utils.eventIsLinkClick(e)) { e.preventDefault(); this.props.setSearchQuery(null); @@ -59,18 +59,34 @@ class Header extends React.Component { rel="noopener noreferrer" style={{ opacity: this.props.searchQuery ? 0.5 : 0, - transition: "opacity ease-in 300ms" + transition: "opacity ease-in 300ms", }} > + {/* + + + */} this.props.setSearchQuery(e.target.value)} + onChange={(e) => this.props.setSearchQuery(e.target.value)} value={this.props.searchQuery || ""} placeholder={"Search..."} - ref={node => { + ref={(node) => { this._inputRef = node; }} /> @@ -93,11 +109,12 @@ class Header extends React.Component { } } -const mapStateToProps = state => ({ - searchQuery: Selectors.getSearchQuery(state) +const mapStateToProps = (state) => ({ + searchQuery: Selectors.getSearchQuery(state), + scale: state.scale, }); -const mapDispatchToProps = dispatch => ({ +const mapDispatchToProps = (dispatch) => ({ setSearchQuery(query) { dispatch(Actions.searchQueryChanged(query)); }, @@ -106,9 +123,9 @@ const mapDispatchToProps = dispatch => ({ }, requestedAboutPage() { dispatch(Actions.requestedAboutPage()); - } + }, + setScale(scale) { + dispatch({ type: "SET_SCALE", scale }); + }, }); -export default connect( - mapStateToProps, - mapDispatchToProps -)(Header); +export default connect(mapStateToProps, mapDispatchToProps)(Header); diff --git a/src/algolia.js b/src/algolia.js index 3a1d371d..a0574bc3 100644 --- a/src/algolia.js +++ b/src/algolia.js @@ -8,13 +8,13 @@ export function search(query, options = {}) { index.search( { query, - attributes: ["objectID", "fileName", "color"], + attributes: ["objectID", "fileName", "color", "nsfw"], attributesToHighlight: [], hitsPerPage: 1000, // https://www.algolia.com/doc/api-reference/api-parameters/typoTolerance/ // min: Retrieve records with the smallest number of typos. typoTolerance: "min", - ...options + ...options, }, (err, content) => { if (err != null) { diff --git a/src/components/Skin.js b/src/components/Skin.js index f8bb4f9c..4a9222c4 100644 --- a/src/components/Skin.js +++ b/src/components/Skin.js @@ -1,4 +1,4 @@ -import React, { useState, useCallback, useMemo } from "react"; +import React, { useState, useCallback } from "react"; import * as Utils from "../utils"; import { SCREENSHOT_HEIGHT } from "../constants"; diff --git a/src/constants.js b/src/constants.js index ede533ba..0bc71002 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,7 +1,4 @@ export const SCREENSHOT_WIDTH = 275; export const SCREENSHOT_HEIGHT = 348; -export const SCALE = 1 / 2; -export const SKIN_WIDTH = SCREENSHOT_WIDTH * SCALE; -export const SKIN_HEIGHT = SCREENSHOT_HEIGHT * SCALE; -export const SKIN_RATIO = SKIN_HEIGHT / SKIN_WIDTH; +export const SKIN_RATIO = SCREENSHOT_HEIGHT / SCREENSHOT_WIDTH; export const ABOUT_PAGE = "ABOUT_PAGE"; diff --git a/src/queryParser.js b/src/queryParser.js new file mode 100644 index 00000000..b84f0cfd --- /dev/null +++ b/src/queryParser.js @@ -0,0 +1,14 @@ +function parseQuery(query) { + const filters = []; + const newQuery = query.replace(/(-?)filter:(nsfw)/, (_, not, attribute) => { + filters.push(`${attribute}=${not ? 0 : 1}`); + return ""; + }); + const options = {}; + if (filters.length) { + options.filters = filters.join(" AND "); + } + return [newQuery, options]; +} + +export default parseQuery; diff --git a/src/queryParser.test.js b/src/queryParser.test.js new file mode 100644 index 00000000..acfce6d8 --- /dev/null +++ b/src/queryParser.test.js @@ -0,0 +1,13 @@ +import queryParser from "./queryParser"; + +test("Only nsfw", () => { + const [query, options] = queryParser("filter:nsfw"); + expect(query).toBe(""); + expect(options).toEqual({ filters: "nsfw=1" }); +}); + +test("Not nsfw", () => { + const [query, options] = queryParser("-filter:nsfw"); + expect(query).toBe(""); + expect(options).toEqual({ filters: "nsfw=0" }); +}); diff --git a/src/redux/epics.js b/src/redux/epics.js index ac7364b6..acc86578 100644 --- a/src/redux/epics.js +++ b/src/redux/epics.js @@ -5,6 +5,7 @@ import * as Selectors from "./selectors"; import * as Utils from "../utils"; import { filter, switchMap, map, mergeMap } from "rxjs/operators"; import { search } from "../algolia"; +import queryParser from "../queryParser"; const urlChangedEpic = (actions) => actions.pipe( @@ -121,14 +122,16 @@ const searchEpic = (actions) => return of(Actions.gotNewMatchingSkins(null)); } - return from(search(query)).pipe( + const [newQuery, options] = queryParser(query); + + return from(search(newQuery, options)).pipe( map((content) => { const matchingSkins = content.hits.map((hit) => ({ hash: hit.objectID, fileName: hit.fileName, color: hit.color, - // TODO: Index these - nsfw: hit.nsfw, + // TODO: Some records still have float scores not booleans. Ignore those. + nsfw: hit.nsfw === true, })); return Actions.gotNewMatchingSkins(matchingSkins); }) diff --git a/src/redux/reducer.js b/src/redux/reducer.js index b89aa1d0..1238a3b1 100644 --- a/src/redux/reducer.js +++ b/src/redux/reducer.js @@ -11,12 +11,17 @@ const defaultState = { fileExplorerOpen: false, activeContentPage: null, totalNumberOfSkins: null, + scale: 0.5, skinChunkData: { chunkSize: 100, numberOfSkins: 64381, chunkFileNames: [] }, skins: {}, }; export default function reducer(state = defaultState, action) { switch (action.type) { + case "SET_SCALE": { + console.log(action); + return { ...state, scale: action.scale }; + } case "GOT_SKIN_DATA": { return { ...state,