Start working on chunking skin data

.gitignore

@@ -22,3 +22,4 @@ yarn-error.log*
 
 public/screenshots
 metadata/
+src/skinData/

scripts/createSearchIndex.js

@@ -1,8 +1,6 @@
 const path = require("path");
 const fs = require("fs");
-const skins = require("../src/skins.json");
 const algoliasearch = require("algoliasearch");
-const { getSkinMetadata } = require("./utils");
 
 const client = algoliasearch("HQ9I5Z6IM5", "f5357f4070cdb6ed652d9c3feeede89f");
 const index = client.initIndex("Skins");
@@ -38,9 +36,9 @@ async function buildSkinIndex(skin) {
   }
   return {
     objectID: skin.md5,
-    md5,
-    fileName,
-    emails: skin.emails || null,
+    //md5,
+    //fileName,
+    // emails: skin.emails || null,
     readmeText
   };
 }
@@ -48,18 +46,16 @@ async function buildSkinIndex(skin) {
 const indexesPromise = Promise.all(
   Object.values(info)
     .filter(skin => skin.type === "CLASSIC")
-    .map(skin => {
-      return buildSkinIndex(skin);
-    })
+    .filter(skin => skin.readmePath)
+    .map(skin => buildSkinIndex(skin))
 );
 
 async function go() {
-  console.log("Building index");
   const indexes = await indexesPromise;
 
   console.log("Writing index");
   const results = await new Promise((resolve, reject) => {
-    index.saveObjects(indexes, function(err, content) {
+    index.partialUpdateObjects(indexes, function(err, content) {
       if (err != null) reject(err);
       resolve(content);
     });

scripts/findSkins.js

@@ -7,8 +7,7 @@ const info = JSON.parse(
 );
 
 const getFileData = async () => {
-  const fileData = {};
-  await Bluebird.map(
+  const fileData = await Bluebird.map(
     Object.values(info).filter(skin => skin.type === "CLASSIC"),
     async skin => {
       const md5 = skin.md5;
@@ -18,22 +17,43 @@ const getFileData = async () => {
         return;
       }
       if (skin.twitterLikes) {
-        console.log({ skin });
+        // console.log({ skin });
       }
       const fileName = path.basename(filePaths[0]);
-      fileData[md5] = {
+      return {
         color: skin.averageColor,
         favorites: skin.twitterLikes,
-        fileName
+        fileName,
+        md5
       };
     },
     { concurrency: 10 }
   );
-  return fileData;
+  const orderedFileData = fileData.sort((a, b) => {
+    return a.favorites < b.favorites;
+  });
+  return orderedFileData.map(({ favorites, ...rest }) => rest);
 };
 
-console.log("Extracting skin data from cache");
+const CHUNK_SIZE = 100;
 getFileData().then(data => {
-  const json = JSON.stringify(data);
-  fs.writeFileSync("src/skins.json", json, "utf8");
+  let start = 0;
+  const chunkFileNames = [];
+  let chunkNumber = 0;
+  while (start <= data.length) {
+    const end = start + CHUNK_SIZE;
+
+    const chunk = data.slice(start, end);
+    const json = JSON.stringify(chunk);
+    const fileName = `skins-${chunkNumber}.json`;
+    fs.writeFileSync(path.join("src", "skinData", fileName), json, "utf8");
+    chunkFileNames.push(fileName);
+    start = end;
+    chunkNumber++;
+  }
+  const json = JSON.stringify({
+    numberOfSkins: data.length,
+    chunkFileNames
+  });
+  fs.writeFileSync("src/skinData/skins.json", json, "utf8");
 });

src/FocusedSkin.js

@@ -182,8 +182,8 @@ class FocusedSkin extends React.Component {
                   transition: "opacity 0.2s ease-out"
                 }}
                 onLoad={() => this.setState({ previewLoaded: true })}
-                src={Utils.screenshotUrlFromHash(this.props.hash)}
-                alt={Utils.filenameFromHash(this.props.hash)}
+                src={this.props.fileName}
+                alt={this.props.fileName}
               />
             )}
           </div>
@@ -220,11 +220,11 @@ class FocusedSkin extends React.Component {
               </span>
             </div>
           )}
-          {Utils.filenameFromHash(this.props.hash)}
+          {this.props.fileName}
           {" ["}
           <DownloadLink
             href={Utils.skinUrlFromHash(this.props.hash)}
-            download={Utils.filenameFromHash(this.props.hash)}
+            download={this.props.fileName}
           >
             Download
           </DownloadLink>
@@ -255,10 +255,11 @@ class FocusedSkin extends React.Component {
   }
 }
 
-const mapStateToProps = state => ({
+const mapStateToProps = (state, ownProps) => ({
   hash: Selectors.getSelectedSkinHash(state),
   initialPosition: Selectors.getSelectedSkinPosition(state),
-  fileExplorerOpen: Selectors.getFileExplorerOpen(state)
+  fileExplorerOpen: Selectors.getFileExplorerOpen(state),
+  fileName: state.skins[ownProps.md5].fileName
 });
 
 const mapDispatchToProps = dispatch => ({

src/Skin.js

@@ -61,7 +61,7 @@ export default class Skin extends React.Component {
           }}
           className={`screenshot ${this.state.loaded ? "loaded" : ""}`}
           onLoad={this._handleLoad}
-          alt={Utils.filenameFromHash(this.props.hash)}
+          alt={this.props.fileName}
         />
       </a>
     );

src/SkinTable.js

@@ -111,7 +111,7 @@ class SkinTable extends React.Component {
 
   _rangeToRender() {
     const { rowHeight, columnCount } = this._getTableDimensions();
-    const hashes = this.props.skinHashes;
+    const { skinCount } = this.props;
     // Add one since we might be showing half a row at the top, and half a row at the bottom
     const visibleRows = Math.ceil(this.state.windowHeight / rowHeight) + 1;
 
@@ -120,7 +120,7 @@ class SkinTable extends React.Component {
     const firstHashToRender = topRow * columnCount;
     const lastHashToRender = Math.min(
       firstHashToRender + visibleRows * columnCount,
-      hashes.length
+      skinCount
     );
 
     return { start: firstHashToRender, end: lastHashToRender };
@@ -128,6 +128,7 @@ class SkinTable extends React.Component {
 
   render() {
     const { columnWidth, rowHeight, columnCount } = this._getTableDimensions();
+    const { skinCount } = this.props;
     const hashes = this.props.skinHashes;
 
     const overscanRowsDown =
@@ -150,7 +151,7 @@ class SkinTable extends React.Component {
     );
     const lastOverscanHashToRender = Math.min(
       lastHashToRender + overscanRowsDown * columnCount,
-      hashes.length
+      skinCount
     );
 
     const skinElements = [];
@@ -161,27 +162,46 @@ class SkinTable extends React.Component {
       const top = row * rowHeight;
       const column = i % columnCount;
       const left = column * columnWidth;
-      skinElements.push(
-        <Skin
-          href={`https://webamp.org/?skinUrl=https://s3.amazonaws.com/webamp-uploaded-skins/skins/${hash}.wsz`}
-          src={Utils.screenshotUrlFromHash(hash)}
-          key={hash}
-          hash={hash}
-          top={top}
-          left={left}
-          width={columnWidth}
-          height={rowHeight}
-          color={this.props.skins[hash].color}
-          isOverscan={isOverscan}
-          selectSkin={this._handleSelectSkin}
-        />
-      );
+      if (hash == null) {
+        // TODO: Pick a random color?
+        // TODO: What happens if they click?
+        skinElements.push(
+          <div
+            key={`placeholder-${i}`}
+            style={{
+              top,
+              left,
+              width: columnWidth,
+              height: rowHeight,
+              position: "absolute",
+              backgroundColor: "magenta"
+            }}
+          />
+        );
+      } else {
+        skinElements.push(
+          <Skin
+            href={`https://webamp.org/?skinUrl=https://s3.amazonaws.com/webamp-uploaded-skins/skins/${hash}.wsz`}
+            src={Utils.screenshotUrlFromHash(hash)}
+            key={hash}
+            hash={hash}
+            top={top}
+            left={left}
+            width={columnWidth}
+            height={rowHeight}
+            color={this.props.skins[hash].color}
+            isOverscan={isOverscan}
+            selectSkin={this._handleSelectSkin}
+            fileName={this.props.skins[hash].fileName}
+          />
+        );
+      }
     }
     return (
       <div
         id="infinite-skins"
         style={{
-          height: Math.ceil(hashes.length / columnCount) * SKIN_HEIGHT,
+          height: Math.ceil(this.props.skinCount / columnCount) * SKIN_HEIGHT,
           position: "relative"
         }}
       >
@@ -194,6 +214,8 @@ class SkinTable extends React.Component {
 const mapStateToProps = state => ({
   skinHashes: Selectors.getMatchingSkinHashes(state),
   skins: Selectors.getSkins(state),
+  // TODO: #lazy Get the length from the chunk metadata
+  skinCount: state.skinChunkData.numberOfSkins,
   selectedSkinHash: Selectors.getSelectedSkinHash(state)
 });
 

src/redux/reducer.js

@@ -1,6 +1,12 @@
-import skins from "../skins.json";
+import skinChunkData from "../skinData/skins.json";
+import firstSkinChunk from "../skinData/skins-0.json";
 import { ABOUT_PAGE } from "../constants";
 
+const skins = {};
+firstSkinChunk.forEach(skin => {
+  skins[skin.md5] = skin;
+});
+
 const defaultState = {
   searchQuery: null,
   selectedSkinPosition: null,
@@ -10,7 +16,8 @@ const defaultState = {
   focusedSkinFile: null,
   fileExplorerOpen: false,
   activeContentPage: null,
-  skins
+  skins,
+  skinChunkData
 };
 
 export default function reducer(state = defaultState, action) {

src/utils.js

@@ -1,19 +1,14 @@
-import skins from "./skins.json";
-
 export function screenshotUrlFromHash(hash) {
   return `https://s3.amazonaws.com/webamp-uploaded-skins/screenshots/${hash}.png`;
 }
 
-export function filenameFromHash(hash) {
-  return skins[hash].fileName;
-}
-
 export function skinUrlFromHash(hash) {
   return `https://s3.amazonaws.com/webamp-uploaded-skins/skins/${hash}.wsz`;
 }
 
 export function getPermalinkUrlFromHash(hash) {
-  return `/skin/${hash}/${filenameFromHash(hash)}/`;
+  // TODO: Make this a selector
+  return `/skin/${hash}/${hash}/`;
 }
 
 export function getAbsolutePermalinkUrlFromHash(hash) {