Progress on maki stuff

experiments/maki-interpreter/command.js

@@ -0,0 +1,136 @@
+const { COMMANDS } = require("./constants");
+
+class Variable {
+  constructor(a, b, name) {
+    this.a = a;
+    this.b = b;
+    this.name = name;
+  }
+}
+
+function decodeUInt32(code) {
+  // prettier-ignore
+  return (
+    (code.charCodeAt(3) << 24) +
+    (code.charCodeAt(2) << 16) +
+    (code.charCodeAt(1) << 8) +
+    code.charCodeAt(0)
+  );
+}
+
+function decodeInt32(code) {
+  let num = decodeUInt32(code);
+  if (num > 0x7fffffff) {
+    num = num - 0xffffffff - 1;
+  }
+  return num;
+}
+
+class Command {
+  constructor({
+    functionsCode,
+    pos,
+    types,
+    variables,
+    functionNames,
+    localFunctions
+  }) {
+    const opcode = functionsCode.charCodeAt(pos);
+    this.pos = pos;
+    this.opcode = opcode;
+    this.arguments = [];
+    this.command = COMMANDS[opcode];
+
+    if (this.command == null) {
+      this.command = {
+        name: `unknown ${opcode}`,
+        in: 0,
+        out: 0
+      };
+      return;
+    }
+
+    if (this.command.arg == null) {
+      this.size = 1;
+      return;
+    }
+
+    const argType = this.command.arg;
+    let arg = null;
+    switch (argType) {
+      case "var": {
+        const variable = decodeInt32(functionsCode.substr(pos + 1, 4));
+        if (variables[variable] != null) {
+          arg = variables[variable];
+        } else {
+          arg = new Variable(-1, -0, "unknown");
+        }
+        this.name = `Unknown ${variable}`;
+        break;
+      }
+      case "line": {
+        const variable = decodeInt32(functionsCode.substr(pos + 1, 4)) + 5;
+        arg = { name: variable, line: variable };
+        break;
+      }
+      case "objFunc": {
+        const variable = decodeInt32(functionsCode.substr(pos + 1, 4));
+        if (functionNames[variable] != null) {
+          arg = functionNames[variable].function;
+        } else {
+          arg = {
+            name: `unknown Object function ${variable}`,
+            parameters: [],
+            return: "unknown"
+          };
+        }
+        break;
+      }
+      case "func": {
+        // Note in the perl code here: "todo, something strange going on here..."
+        const variable = decodeInt32(functionsCode.substr(pos + 1, 4)) + 5;
+        const s = functionsCode.substr(pos + 1, 1);
+        console.log(functionsCode.length);
+        console.log(s.charCodeAt(0));
+        arg = {
+          name: `func${variable + pos}`,
+          code: [],
+          offset: variable + pos
+        };
+        if (opcode === 25) {
+          // console.log(variable);
+        }
+        if (localFunctions[variable + pos] == null) {
+          localFunctions[variable + pos] = {
+            function: arg,
+            offset: variable + pos
+          };
+        }
+        break;
+      }
+      case "obj": {
+        const variable = decodeInt32(functionsCode.substr(pos + 1, 4));
+        arg = types[variable];
+        break;
+      }
+    }
+
+    this.arguments = [arg];
+    this.size = 5;
+
+    /*
+    # look forward for a stack protection block
+    # (why do I have to look FORWARD. stupid nullsoft)
+    if( length($code) > $pos+5+4 &&
+        decodeUInt32( substr( $code, $pos+5, 4 ) ) >= 0xffff0000 ) {
+      $self->{size} += 4;
+    }
+    */
+
+    if (opcode === 112) {
+      this.size += 1;
+    }
+  }
+}
+
+module.exports = Command;

experiments/maki-interpreter/index.js

@@ -1,6 +1,7 @@
 const { COMMANDS } = require("./constants");
+const Command = require("./command");
 const MAGIC = "FG";
-const ENCODING = "utf8";
+const ENCODING = "ascii";
 
 class Parser {
   _readMagic() {
@@ -102,6 +103,11 @@ class Parser {
     while (i < code.length) {
       const opCode = code.charCodeAt(i);
       const command = COMMANDS[opCode];
+      if (command == null) {
+        // console.warn(`Missing command opCode: ${opCode}`);
+        i++;
+        continue;
+      }
       if (command.arg == null) {
         i++;
         continue;
@@ -152,6 +158,37 @@ class Parser {
     return this._readStringOfLength(this._readUInt16LE());
   }
 
+  _decodeCode({ functionsCode, types, variables, functionNames, functions }) {
+    let pos = 0;
+    const localFunctions = {};
+    const results = [];
+    const poss = [];
+    while (pos < functionsCode.length) {
+      poss.push(pos);
+      const command = new Command({
+        functionsCode,
+        pos,
+        types,
+        variables,
+        functionNames,
+        localFunctions
+      });
+      pos += command.size;
+      results.push(command);
+    }
+    // TODO: Don't mutate
+    Object.values(localFunctions).forEach(localFunction => {
+      functions.push(localFunction);
+    });
+
+    functions.sort((a, b) => {
+      // TODO: Confirm that I have this the right way round
+      return a.offset - b.offset;
+    });
+
+    return results;
+  }
+
   parse(buffer) {
     this._buffer = buffer;
     this._i = 0;
@@ -165,6 +202,13 @@ class Parser {
     const constants = this._readConstants();
     const functions = this._readFunctions();
     const functionsCode = this._readFunctionsCode();
+    const decoding = this._decodeCode({
+      functionsCode,
+      types,
+      variables,
+      functionNames,
+      functions
+    });
     this._readCommands(functionsCode);
     return {
       magic,
@@ -173,7 +217,8 @@ class Parser {
       variables,
       constants,
       functionsCode,
-      functions
+      functions,
+      decoding
     };
   }
 }

experiments/maki-interpreter/index.test.js

@@ -8,26 +8,26 @@ function parseFile(relativePath) {
 }
 
 describe("standardframe.maki", () => {
-  let debug;
+  let maki;
   beforeEach(() => {
-    debug = parseFile("./fixtures/standardframe.maki");
+    maki = parseFile("./fixtures/standardframe.maki");
   });
 
   test("can read magic", () => {
-    expect(debug.magic).toBe("FG");
+    expect(maki.magic).toBe("FG");
   });
 
   test("can read byte code", () => {
-    expect(debug.types.length).toBe(33);
-    expect(debug.types[0]).toBe("516549714a510d87b5a6e391e7f33532");
-    debug.types.forEach(type => {
+    expect(maki.types.length).toBe(33);
+    expect(maki.types[0]).toBe("516549714a510d87b5a6e391e7f33532");
+    maki.types.forEach(type => {
       expect(type.length).toBe(32);
     });
   });
 
   test("can read functionNames", () => {
-    expect(debug.functionNames.length).toBe(12);
-    expect(debug.functionNames.map(func => func.name)).toEqual([
+    expect(maki.functionNames.length).toBe(12);
+    expect(maki.functionNames.map(func => func.name)).toEqual([
       "onScriptLoaded",
       "getScriptGroup",
       "getParam",
@@ -41,31 +41,41 @@ describe("standardframe.maki", () => {
       "newGroup",
       "init"
     ]);
-    debug.functionNames.forEach(func => {
-      expect(debug.types[func.classType]).not.toBe(undefined);
+    maki.functionNames.forEach(func => {
+      expect(maki.types[func.classType]).not.toBe(undefined);
     });
   });
 
   test("can read variables", () => {
-    expect(debug.variables.length).toBe(56);
-    debug.variables.forEach(variable => {
-      expect(debug.types[variable.type]).not.toBe(undefined);
+    expect(maki.variables.length).toBe(56);
+    maki.variables.forEach(variable => {
+      expect(maki.types[variable.type]).not.toBe(undefined);
     });
   });
 
   test("can read constants", () => {
-    expect(debug.constants.length).toBe(23);
+    expect(maki.constants.length).toBe(23);
   });
 
   test("can read functions", () => {
-    expect(debug.functions).toEqual([
+    // console.log(maki.functions);
+    expect(maki.functions).toEqual([
       { varNum: 0, offset: 0, funcNum: 0 },
       { varNum: 0, offset: 296, funcNum: 4 },
+      { function: { code: [], name: "func332", offset: 332 }, offset: 332 },
       { varNum: 2, offset: 559, funcNum: 9 }
     ]);
   });
 
   test("can read function code", () => {
-    expect(debug.functionsCode.length).toBe(1004);
+    expect(maki.functionsCode.length).toBe(1004);
+  });
+
+  test("can read function code", () => {
+    expect(maki.functionsCode.length).toBe(1004);
+  });
+
+  test("can read decoding", () => {
+    expect(maki.decoding.length).toBe(256);
   });
 });

experiments/maki-interpreter/package.json

@@ -5,13 +5,17 @@
   "author": "Jordan Eldredge <jordan@jordaneldredge.com>",
   "license": "MIT",
   "scripts": {
-    "test": "jest",
+    "test": "jest -u",
     "tdd": "jest --watch",
     "decompile": "cd reference/maki_decompiler_1.1 && perl mdc.pl ../../fixtures/standardframe.maki"
   },
   "devDependencies": {
-    "jest": "^23.6.0",
+    "glob": "^7.1.4",
+    "jest": "^24.8.0",
     "prettier": "^1.15.3"
   },
-  "prettier": {}
+  "prettier": {},
+  "dependencies": {
+    "jszip": "^3.2.1"
+  }
 }

experiments/maki-interpreter/tools/extract-functions.js

@@ -0,0 +1,48 @@
+const fs = require("fs");
+const { parse } = require("../");
+const JSZip = require("jszip");
+
+async function getFunctionNames(absolutePath) {
+  const buffer = fs.readFileSync(absolutePath);
+  const zip = await JSZip.loadAsync(buffer);
+  const files = zip.file(/\.maki$/);
+  const buffers = await Promise.all(
+    files.map(file => file.async("nodebuffer"))
+  );
+  const functionNames = buffers.reduce((_functionNames, buf) => {
+    const maki = parse(buf);
+    maki.functionNames.forEach(func => {
+      const className = maki.types[func.classType];
+      _functionNames.add(`${className}::${func.name}`);
+    });
+    return _functionNames;
+  }, new Set());
+  return functionNames;
+}
+
+async function main(paths) {
+  const results = await Promise.all(
+    paths.map(async path => {
+      const functionNames = await getFunctionNames(path);
+      return { path, functionNames };
+    })
+  );
+  const functionCounts = {};
+  results.forEach(skin => {
+    skin.functionNames.forEach(name => {
+      const originalCount = functionCounts[name];
+      functionCounts[name] = (originalCount || 0) + 1;
+    });
+  });
+  console.log({
+    totalFunctionCount: Object.keys(functionCounts).length
+    // functionCounts
+  });
+}
+
+const paths = [
+  "/Volumes/Mobile Backup/skins/skins/dump/Stylish/micro/micro.wal",
+  "/Volumes/Mobile Backup/skins/skins/random/Winamp Skins/Skins/WTF/Almin_Agic_Skin.wal"
+];
+
+main(paths);

experiments/maki-interpreter/tools/extract-maki-files.js

@@ -0,0 +1,53 @@
+#!/usr/bin/env node
+
+const path = require("path");
+const fs = require("fs");
+const JSZip = require("jszip");
+const glob = require("glob");
+
+function findWals(parentDir) {
+  return new Promise((resolve, reject) => {
+    // options is optional
+    glob("**/*.wal", { cwd: parentDir }, (err, files) => {
+      if (err) {
+        return reject(err);
+      }
+      resolve(files);
+    });
+  });
+}
+
+async function main(skinPath) {
+  // Find all the .wal files in the path recursively
+  const wals = await findWals(skinPath);
+  const zips = await Promise.all(
+    wals.map(async skin => {
+      const walPath = path.join(skinPath, skin);
+
+      const buffer = fs.readFileSync(walPath);
+      const zip = await JSZip.loadAsync(buffer);
+      const makis = await Promise.all(
+        zip.file(/\.maki$/i).map(async file => {
+          return {
+            name: file.name,
+            buffer: await file.async("nodebuffer")
+          };
+        })
+      );
+      return {
+        path: walPath,
+        makis
+      };
+    })
+  );
+  console.log(zips);
+  // For each skin
+  //  extract it
+  //  find all `.maki` files
+  //  get its md5
+  //
+  //  create a directory for its
+  // For each sk
+}
+
+main("/Volumes/Mobile Backup/skins/skins/dump/Compact-Utility");