Added documentation

README.md

@@ -2,3 +2,74 @@
 
 A set of tools for matching opcodes across different versions of the FFXIV
 binary.
+
+## Prerequisites
+
+1. Ensure you have python >= 3.7 installed.
+
+1. Set up a python venv:
+   ```sh
+   python -m venv /path/to/venv/dir
+   ```
+
+1. Activate the venv
+
+   Linux:
+   ```sh
+   source venv/bin/activate # csh or fish variants available as well
+   ```
+
+   Windows:
+   ```
+   venv\Scripts\activate
+   ```
+
+1. Install dependencies
+
+   ```sh
+   pip install -r requirements.txt
+   ```
+
+1. Ensure you have [radare](https://github.com/radareorg/radare2) somewhere on
+    your path
+
+1. For `traces-diff.py`, an NVIDIA GPU and CUDA support are highly recommended,
+   but not required. It might just take twice as long to train.
+
+## Usage
+
+Pass `--help` to any of the scripts for usage.
+
+### Workflow for minor patches
+
+Example:
+```sh
+python minor_patch_diff.py ffxiv_dx11.6.30.exe ffxiv_dx11.6.30h.exe > 6.30h.diff.json
+```
+
+Post-diff processing:
+```sh
+python generate_opcodes_file.py 6.30h 6.30h.diff.json Ipcs.6.30h.h
+# TODO: Need a script to resolve opcodes for handlers with multiple opcodes
+python generate_act_format.py Ipcs.6.30h.h
+```
+
+### Workflow for major patches
+
+We'll need to generate "traces" as signatures for every packet handler.
+
+Example:
+```sh
+python generate_deep_traces.py ffxiv_dx11.6.28h.exe 6.28h-traces
+python generate_deep_traces.py ffxiv_dx11.6.30.exe 6.30-traces
+python traces_diff.py 6.28h-traces 6.30-traces 6.30.diff.json
+```
+
+Post-diff processing:
+```sh
+python generate_opcodes_file.py 6.30h 6.30h.diff.json Ipcs.6.30h.h
+# TODO: Need a script to resolve opcodes for handlers with multiple opcodes
+python generate_act_format.py Ipcs.6.30h.h
+```
+
+###

generate_act_format.py

@@ -34,6 +34,16 @@ desired_names = {
 @click.command()
 @click.argument("opcodes_file", type=click.File("r"))
 def generate_act_format(opcodes_file):
+    """
+    Parses an OPCODES_FILE and outputs opcodes in a format expected by ACT.
+    This is also just a bunch of regexes slapped together.
+
+    Outputs to stdout.
+
+    Example:
+
+    python generate_act_format.py Ipcs.h
+    """
     opcode_mapping = dict()
 
     for line in opcodes_file.readlines():

generate_deep_traces.py

@@ -209,6 +209,20 @@ def trace_lines(blocks, ref):
 )
 @click.argument("output_dir", type=click.Path(file_okay=False))
 def generate_deep_traces(exe_file, output_dir):
+    """
+    Generates deep traces for every packet handler in the target EXE_FILE.
+    This outputs an ASM trace as an .asm file for each pointer opcode.
+    It also outputs an `opcode_sets.json` that maps each pointer opcode to the
+    full set of opcodes handled by that trace.
+
+    These aren't normal traces by any measure; they are simply basic blocks
+    printed in BFS order, where children blocks are added to the BFS tree
+    by traversing calls and jumps.
+
+    Example:
+
+    python generate_deep_traces.py ffxiv_dx11.6.28h.exe 6.28h-traces
+    """
     opcodes_db, blocks = extract_opcode_data(exe_file)
 
     pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)

generate_opcodes_file.py

@@ -49,8 +49,20 @@ def replace_line_with_new_opcode(line, diff, ver):
 @click.argument("new_version_string")
 @click.argument("diff_file", type=click.File("r"))
 @click.argument("opcodes_file", type=click.File("r"))
-@click.option("--reverse", is_flag=True)
+@click.option(
+    "--reverse", is_flag=True, help="Applies the diff file in the opposite direction"
+)
 def generate_opcodes_file(new_version_string, diff_file, opcodes_file, reverse):
+    """
+    Parses an OPCODES_FILE and applies a JSON DIFF_FILE to generate a new one.
+    The opcodes file is basically anything that has syntax resembling Sapphire's
+    `Ipcs.h`. It doesn't do any C++ header parsing; it's just a bunch of regexes
+    slapped together but it works.
+
+    Example:
+
+    python generate_opcodes_file.py 6.30h diff.json Ipcs.h
+    """
     diff = load_diff_file(diff_file, reverse)
     queued_lines = []
     for line in opcodes_file.readlines():

minor_patch_diff.py

@@ -190,6 +190,26 @@ def find_opcode_matches(old_opcodes_db, new_opcodes_db):
     "new_exe", type=click.Path(exists=True, dir_okay=False, resolve_path=True)
 )
 def minor_patch_diff(old_exe, new_exe):
+    """
+    Generates an opcode diff file for minor patches (e.g 6.30 => 6.30h).
+
+    This script outputs to stdout, so pipe it to a json file.
+
+    The format of the output is a list (all fields are optional):
+
+    \b
+    [
+        {
+            "old": (list of opcodes in the switch case),
+            "new": (list of opcodes in the switch case),
+        },
+        ...
+    ]
+
+    Example:
+
+    python minor_patch_diff.py ffxiv_dx11.old.exe ffxiv_dx11.new.exe > diff.json
+    """
     old_opcodes_db = extract_opcode_data(old_exe)
     new_opcodes_db = extract_opcode_data(new_exe)
 

traces_diff.py

@@ -187,23 +187,6 @@ class OpcodeMatcher:
         opcodes where the confidence is greater than the given threshold.
         2. Old opcodes for which a match could not be confidently found.
         3. New opcodes for which a match could not be confidently found.
-
-        The format of the output is a list (all fields are optional):
-        [
-            {
-                "old": (list of opcodes in the switch case),
-                "new": (list of opcodes in the switch case),
-                "score_lead": (confidence above 2nd best match),
-                "unknown": (true if a match was not made in this case),
-                "candidates: [
-                    {
-                        "set": (list of opcodes in switch case),
-                        "score": (candidate score),
-                    }
-                    ...
-                ]
-            }
-        ]
         """
         output = []
 
@@ -275,6 +258,36 @@ def print_banner(text):
 )
 @click.argument("output_file", type=click.Path(dir_okay=False, resolve_path=True))
 def traces_diff(old_traces, new_traces, output_file):
+    """
+    Compares the OLD_TRACES and NEW_TRACES directories generated by the
+    `generate_deep_traces.py` script.
+
+    Creates a JSON OUTPUT_FILE containing best matches and opcodes with
+    ambiguous candidates that did not yield a definite match.
+
+    The format of the output is a list (all fields are optional):
+
+    \b
+    [
+        {
+            "old": (list of opcodes in the switch case),
+            "new": (list of opcodes in the switch case),
+            "score_lead": (confidence above 2nd best match),
+            "unknown": (true if a match was not made in this case),
+            "candidates: [
+                {
+                    "set": (list of opcodes in switch case),
+                    "score": (candidate score),
+                },
+                ...
+            ]
+        },
+        ...
+    ]
+    Example:
+
+    python traces_diff.py old-traces/ new-traces/ diff.json
+    """
     tokens = Tokens()
     old_trace_data = TraceData.load_data(old_traces, tokens)
     new_trace_data = TraceData.load_data(new_traces, tokens)