Mirrors/webamp
1
0
Fork 0
mirror of https://github.com/captbaritone/webamp.git synced 2026-07-23 10:07:35 +00:00
Code Issues Projects Releases Wiki Activity

Query parsing and cleanup

Browse source
This commit is contained in:
Jordan Eldredge 2020-06-12 20:58:28 -07:00
parent 7e0fe1f38f
commit bd31f04455
14 changed files with 127 additions and 247 deletions
Download patch file Download diff file Expand all files Collapse all files

49
nodes.md Normal file
View file

@ -0,0 +1,49 @@
# States
1. ~Lazy loading initial results~
2. Above the fold placeholders
3. Above the fold images
4. Scrolling existing placeholders
5. Scrolling unfetched placeholders
Show random colors
6. Clicked scrollbar and went to the bottom of the page
7. Cliked an image
We should include the first ~100 results in the initial payload
Loading a set of search results
1. Include teh color and md5 in the results
Each segment should have a state:
- Loaded
- Loading
- Errored
- Not Requested
When we want to show a placeholder we:
- Check if we already have it
- If so, we show it
- If not, pick a random color, and then (asycn) we check which segment it would be in
- If that segement is loading, noop
- If the segment has not been requested, we request it and mark it as loading
- When a segment comes back, we update our state
When a user clicks a skin
- If the skin is loaded (we have the md5) we take the user to the permalink
- If the skin is not loaded, we check which segment it would be in
- If the segment has not yet been requested (unlikely, since it should have been requested when it came into view) we request it
- If the segment is loading, we await its completion.
- If 500ms passes before the load completes, we show a spinnner or something
- When the loading completes we take the user to the permalink
## Functions we'll need
- `getSegmentForIndex(i)`
- `getMd5ForIndex(i)` (syncronous, but may kick off an async action)
- `getColorForIndex(i)` (syncronus. May give a BS result if it does not have one right away, may kick off an async action)
- `getSegmentStatus(segmentId)`
- `awaitMd5ForIndex(i)`

71
scripts/createSearchIndex.js
View file

@ -1,71 +0,0 @@
const path = require("path");
const fs = require("fs");
const algoliasearch = require("algoliasearch");
const client = algoliasearch("HQ9I5Z6IM5", "f5357f4070cdb6ed652d9c3feeede89f");
const index = client.initIndex("Skins");
const CACHE_PATH = "/Volumes/Mobile Backup/skins/cache/";
const info = JSON.parse(
fs.readFileSync(path.join(CACHE_PATH, "info.json"), "utf8")
);
function tuncate(str, len) {
const overflow = str.length - len;
if (overflow < 0) {
return str;
}
const half = Math.floor((len - 1) / 2);
const start = str.slice(0, half);
const end = str.slice(-half);
return `${start} ########### ${end}`;
}
async function buildSkinIndex(skin) {
const { md5, filePaths } = skin;
if (!filePaths || filePaths.length === 0) {
console.warn("no file name for ", md5);
return;
}
const fileName = path.basename(filePaths[0]);
let readmeText = null;
if (skin.readmePath) {
readmeText = tuncate(fs.readFileSync(skin.readmePath, "utf8"), 4800);
}
return {
objectID: skin.md5,
//md5,
//fileName,
// emails: skin.emails || null,
// readmeText
// color: skin.averageColor
// twitterLikes: Number(skin.twitterLikes || 0)
};
}
const indexesPromise = Promise.all(
Object.values(info)
.filter((skin) => skin.type === "CLASSIC")
.map((skin) => buildSkinIndex(skin))
);
async function go({ dry = true }) {
const indexes = await indexesPromise;
if (dry) {
console.log("Index turned off. Turn it on if you really mean it");
return;
}
console.log("Writing index");
const results = await new Promise((resolve, reject) => {
index.partialUpdateObjects(indexes, function (err, content) {
if (err != null) reject(err);
resolve(content);
});
});
console.log("done!", results);
}
go({ dry: false }); // .then(content => console.log("Updated index for:", content.length));

77
scripts/findSkins.js
View file

@ -1,77 +0,0 @@
const fs = require("fs");
const path = require("path");
const Bluebird = require("bluebird");
const info = JSON.parse(
fs.readFileSync("/Volumes/Mobile Backup/skins/cache/info.json", "utf8")
);
const getFileData = async () => {
const fileData = await Bluebird.map(
Object.values(info).filter(skin => skin.type === "CLASSIC"),
async skin => {
const md5 = skin.md5;
const filePaths = skin.filePaths;
if (!filePaths || filePaths.length === 0) {
console.warn("no file name for ", md5);
return;
}
if (skin.twitterLikes) {
// console.log({ skin });
}
const fileName = path.basename(filePaths[0]);
return {
color: skin.averageColor,
favorites: skin.twitterLikes || 0,
fileName,
md5
};
},
{ concurrency: 10 }
);
const orderedFileData = fileData.sort((a, b) => {
return b.favorites - a.favorites;
});
return orderedFileData.map(({ favorites, ...rest }) => rest);
};
const SKIN_DATA_ROOT = "public/skinData";
const BOOTSTRAP_SKIN_DATA_ROOT = "src";
const CHUNK_SIZE = 500;
getFileData().then(data => {
let start = 0;
const chunkFileNames = [];
let chunkNumber = 0;
while (start <= data.length) {
const end = start + CHUNK_SIZE;
const chunk = data.slice(start, end);
const json = JSON.stringify(chunk);
const fileName = `skins-${chunkNumber}.json`;
fs.writeFileSync(path.join(SKIN_DATA_ROOT, fileName), json, "utf8");
if (chunkNumber === 0) {
fs.writeFileSync(
path.join(BOOTSTRAP_SKIN_DATA_ROOT, fileName),
json,
"utf8"
);
}
chunkFileNames.push(fileName);
start = end;
chunkNumber++;
}
const json = JSON.stringify(
{
chunkSize: CHUNK_SIZE,
numberOfSkins: data.length,
chunkFileNames
},
null,
2
);
fs.writeFileSync(
path.join(BOOTSTRAP_SKIN_DATA_ROOT, "skins.json"),
json,
"utf8"
);
});

63
scripts/utils.js
View file

@ -1,63 +0,0 @@
const path = require("path");
const fs = require("fs");
const md5File = require("md5-file");
var { exec } = require("child_process");
const METADATA_ROOT = path.join(__dirname, "../metadata/");
function getSkinMetadataPath(md5, key) {
return path.join(METADATA_ROOT, md5, `${key}.json`);
}
function getSkinMetadata(md5, key) {
const skinMetadataPath = getSkinMetadataPath(md5, key);
return new Promise((resolve, reject) => {
fs.readFile(skinMetadataPath, "utf8", (err, data) => {
if (err != null) reject(err);
resolve(JSON.parse(data));
});
});
}
function ensureDirectoryExistence(filePath) {
var dirname = path.dirname(filePath);
if (fs.existsSync(dirname)) {
return true;
}
ensureDirectoryExistence(dirname);
fs.mkdirSync(dirname);
}
async function writeSkinMetadata(md5, key, data) {
const skinMetadataPath = getSkinMetadataPath(md5, key);
ensureDirectoryExistence(skinMetadataPath);
return new Promise((resolve, reject) => {
fs.writeFile(
skinMetadataPath,
JSON.stringify(data, null, 2),
"utf8",
err => {
if (err != null) {
reject(err);
}
resolve(skinMetadataPath);
}
);
});
}
function getFileMd5(filePath) {
return new Promise((resolve, reject) => {
md5File(filePath, (err, hash) => {
if (err) reject(err);
resolve(hash);
});
});
}
module.exports = {
getSkinMetadata,
writeSkinMetadata,
getFileMd5
};

10
src/App.js
View file

@ -9,12 +9,12 @@ import FocusedSkin from "./FocusedSkin";
import * as Selectors from "./redux/selectors";
import { ABOUT_PAGE } from "./constants";
import * as Utils from "./utils";
import { SKIN_WIDTH, SKIN_RATIO } from "./constants";
import { SCREENSHOT_WIDTH, SKIN_RATIO } from "./constants";
// Render your table
const getTableDimensions = (windowWidth) => {
const columnCount = Math.floor(windowWidth / SKIN_WIDTH);
const getTableDimensions = (windowWidth, scale) => {
const columnCount = Math.floor(windowWidth / (SCREENSHOT_WIDTH * scale));
const columnWidth = windowWidth / columnCount; // TODO: Consider flooring this to get things aligned to the pixel
const rowHeight = columnWidth * SKIN_RATIO;
return { columnWidth, rowHeight, columnCount };
@ -36,7 +36,8 @@ function useWindowSize() {
function App(props) {
const { windowWidth, windowHeight } = useWindowSize();
const { columnWidth, rowHeight, columnCount } = getTableDimensions(
windowWidth
windowWidth,
props.scale
);
return (
<div>
@ -73,6 +74,7 @@ const mapStateToProps = (state) => ({
selectedSkinHash: Selectors.getSelectedSkinHash(state),
overlayShouldAnimate: Selectors.overlayShouldAnimate(state),
aboutPage: Selectors.getActiveContentPage(state) === ABOUT_PAGE,
scale: state.scale,
});
export default connect(mapStateToProps)(App);

9
src/App.test.js
View file

@ -1,9 +0,0 @@
import React from "react";
import ReactDOM from "react-dom";
import App from "./App";
it("renders without crashing", () => {
const div = document.createElement("div");
ReactDOM.render(<App />, div);
ReactDOM.unmountComponentAtNode(div);
});

43
src/Header.js
View file

@ -14,7 +14,7 @@ class Header extends React.Component {
}
componentDidMount() {
const handler = e => {
const handler = (e) => {
// slash
if (e.keyCode === 191) {
if (this._inputRef == null) {
@ -41,7 +41,7 @@ class Header extends React.Component {
<h1>
<a
href="/"
onClick={e => {
onClick={(e) => {
if (Utils.eventIsLinkClick(e)) {
e.preventDefault();
this.props.setSearchQuery(null);
@ -59,18 +59,34 @@ class Header extends React.Component {
rel="noopener noreferrer"
style={{
opacity: this.props.searchQuery ? 0.5 : 0,
transition: "opacity ease-in 300ms"
transition: "opacity ease-in 300ms",
}}
>
<AlgoliaLogo />
</a>
{/*
<button
onClick={() => {
this.props.setScale(this.props.scale + 0.1);
}}
>
+
</button>
<button
onClick={() => {
this.props.setScale(this.props.scale - 0.1);
}}
>
-
</button>
*/}
<input
style={{ marginLeft: 10 }}
type="text"
onChange={e => this.props.setSearchQuery(e.target.value)}
onChange={(e) => this.props.setSearchQuery(e.target.value)}
value={this.props.searchQuery || ""}
placeholder={"Search..."}
ref={node => {
ref={(node) => {
this._inputRef = node;
}}
/>
@ -93,11 +109,12 @@ class Header extends React.Component {
}
}
const mapStateToProps = state => ({
searchQuery: Selectors.getSearchQuery(state)
const mapStateToProps = (state) => ({
searchQuery: Selectors.getSearchQuery(state),
scale: state.scale,
});
const mapDispatchToProps = dispatch => ({
const mapDispatchToProps = (dispatch) => ({
setSearchQuery(query) {
dispatch(Actions.searchQueryChanged(query));
},
@ -106,9 +123,9 @@ const mapDispatchToProps = dispatch => ({
},
requestedAboutPage() {
dispatch(Actions.requestedAboutPage());
}
},
setScale(scale) {
dispatch({ type: "SET_SCALE", scale });
},
});
export default connect(
mapStateToProps,
mapDispatchToProps
)(Header);
export default connect(mapStateToProps, mapDispatchToProps)(Header);

4
src/algolia.js
View file

@ -8,13 +8,13 @@ export function search(query, options = {}) {
index.search(
{
query,
attributes: ["objectID", "fileName", "color"],
attributes: ["objectID", "fileName", "color", "nsfw"],
attributesToHighlight: [],
hitsPerPage: 1000,
// https://www.algolia.com/doc/api-reference/api-parameters/typoTolerance/
// min: Retrieve records with the smallest number of typos.
typoTolerance: "min",
...options
...options,
},
(err, content) => {
if (err != null) {

2
src/components/Skin.js
View file

@ -1,4 +1,4 @@
import React, { useState, useCallback, useMemo } from "react";
import React, { useState, useCallback } from "react";
import * as Utils from "../utils";
import { SCREENSHOT_HEIGHT } from "../constants";

5
src/constants.js
View file

@ -1,7 +1,4 @@
export const SCREENSHOT_WIDTH = 275;
export const SCREENSHOT_HEIGHT = 348;
export const SCALE = 1 / 2;
export const SKIN_WIDTH = SCREENSHOT_WIDTH * SCALE;
export const SKIN_HEIGHT = SCREENSHOT_HEIGHT * SCALE;
export const SKIN_RATIO = SKIN_HEIGHT / SKIN_WIDTH;
export const SKIN_RATIO = SCREENSHOT_HEIGHT / SCREENSHOT_WIDTH;
export const ABOUT_PAGE = "ABOUT_PAGE";

14
src/queryParser.js Normal file
View file

@ -0,0 +1,14 @@
function parseQuery(query) {
const filters = [];
const newQuery = query.replace(/(-?)filter:(nsfw)/, (_, not, attribute) => {
filters.push(`${attribute}=${not ? 0 : 1}`);
return "";
});
const options = {};
if (filters.length) {
options.filters = filters.join(" AND ");
}
return [newQuery, options];
}
export default parseQuery;

13
src/queryParser.test.js Normal file
View file

@ -0,0 +1,13 @@
import queryParser from "./queryParser";
test("Only nsfw", () => {
const [query, options] = queryParser("filter:nsfw");
expect(query).toBe("");
expect(options).toEqual({ filters: "nsfw=1" });
});
test("Not nsfw", () => {
const [query, options] = queryParser("-filter:nsfw");
expect(query).toBe("");
expect(options).toEqual({ filters: "nsfw=0" });
});

9
src/redux/epics.js
View file

@ -5,6 +5,7 @@ import * as Selectors from "./selectors";
import * as Utils from "../utils";
import { filter, switchMap, map, mergeMap } from "rxjs/operators";
import { search } from "../algolia";
import queryParser from "../queryParser";
const urlChangedEpic = (actions) =>
actions.pipe(
@ -121,14 +122,16 @@ const searchEpic = (actions) =>
return of(Actions.gotNewMatchingSkins(null));
}
return from(search(query)).pipe(
const [newQuery, options] = queryParser(query);
return from(search(newQuery, options)).pipe(
map((content) => {
const matchingSkins = content.hits.map((hit) => ({
hash: hit.objectID,
fileName: hit.fileName,
color: hit.color,
// TODO: Index these
nsfw: hit.nsfw,
// TODO: Some records still have float scores not booleans. Ignore those.
nsfw: hit.nsfw === true,
}));
return Actions.gotNewMatchingSkins(matchingSkins);
})

5
src/redux/reducer.js
View file

@ -11,12 +11,17 @@ const defaultState = {
fileExplorerOpen: false,
activeContentPage: null,
totalNumberOfSkins: null,
scale: 0.5,
skinChunkData: { chunkSize: 100, numberOfSkins: 64381, chunkFileNames: [] },
skins: {},
};
export default function reducer(state = defaultState, action) {
switch (action.type) {
case "SET_SCALE": {
console.log(action);
return { ...state, scale: action.scale };
}
case "GOT_SKIN_DATA": {
return {
...state,