From 9d6b4dfa247735105b9b7cccb047e11fc64b90ec Mon Sep 17 00:00:00 2001 From: Flawed <33593723+ff14wed@users.noreply.github.com> Date: Wed, 18 Jan 2023 22:47:40 -0800 Subject: [PATCH] Added documentation --- README.md | 73 +++++++++++++++++++++++++++++++++++++++- generate_act_format.py | 10 ++++++ generate_deep_traces.py | 14 ++++++++ generate_opcodes_file.py | 14 +++++++- minor_patch_diff.py | 20 +++++++++++ traces_diff.py | 47 ++++++++++++++++---------- 6 files changed, 159 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index fbb0937..60f92b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,75 @@ # opcodediff A set of tools for matching opcodes across different versions of the FFXIV -binary. \ No newline at end of file +binary. + +## Prerequisites + +1. Ensure you have python >= 3.7 installed. + +1. Set up a python venv: + ```sh + python -m venv /path/to/venv/dir + ``` + +1. Activate the venv + + Linux: + ```sh + source venv/bin/activate # csh or fish variants available as well + ``` + + Windows: + ``` + venv\Scripts\activate + ``` + +1. Install dependencies + + ```sh + pip install -r requirements.txt + ``` + +1. Ensure you have [radare](https://github.com/radareorg/radare2) somewhere on + your path + +1. For `traces-diff.py`, an NVIDIA GPU and CUDA support are highly recommended, + but not required. It might just take twice as long to train. + +## Usage + +Pass `--help` to any of the scripts for usage. + +### Workflow for minor patches + +Example: +```sh +python minor_patch_diff.py ffxiv_dx11.6.30.exe ffxiv_dx11.6.30h.exe > 6.30h.diff.json +``` + +Post-diff processing: +```sh +python generate_opcodes_file.py 6.30h 6.30h.diff.json Ipcs.6.30h.h +# TODO: Need a script to resolve opcodes for handlers with multiple opcodes +python generate_act_format.py Ipcs.6.30h.h +``` + +### Workflow for major patches + +We'll need to generate "traces" as signatures for every packet handler. + +Example: +```sh +python generate_deep_traces.py ffxiv_dx11.6.28h.exe 6.28h-traces +python generate_deep_traces.py ffxiv_dx11.6.30.exe 6.30-traces +python traces_diff.py 6.28h-traces 6.30-traces 6.30.diff.json +``` + +Post-diff processing: +```sh +python generate_opcodes_file.py 6.30h 6.30h.diff.json Ipcs.6.30h.h +# TODO: Need a script to resolve opcodes for handlers with multiple opcodes +python generate_act_format.py Ipcs.6.30h.h +``` + +### \ No newline at end of file diff --git a/generate_act_format.py b/generate_act_format.py index 8ca4b80..417af31 100644 --- a/generate_act_format.py +++ b/generate_act_format.py @@ -34,6 +34,16 @@ desired_names = { @click.command() @click.argument("opcodes_file", type=click.File("r")) def generate_act_format(opcodes_file): + """ + Parses an OPCODES_FILE and outputs opcodes in a format expected by ACT. + This is also just a bunch of regexes slapped together. + + Outputs to stdout. + + Example: + + python generate_act_format.py Ipcs.h + """ opcode_mapping = dict() for line in opcodes_file.readlines(): diff --git a/generate_deep_traces.py b/generate_deep_traces.py index 1b003e9..eaaa1f3 100644 --- a/generate_deep_traces.py +++ b/generate_deep_traces.py @@ -209,6 +209,20 @@ def trace_lines(blocks, ref): ) @click.argument("output_dir", type=click.Path(file_okay=False)) def generate_deep_traces(exe_file, output_dir): + """ + Generates deep traces for every packet handler in the target EXE_FILE. + This outputs an ASM trace as an .asm file for each pointer opcode. + It also outputs an `opcode_sets.json` that maps each pointer opcode to the + full set of opcodes handled by that trace. + + These aren't normal traces by any measure; they are simply basic blocks + printed in BFS order, where children blocks are added to the BFS tree + by traversing calls and jumps. + + Example: + + python generate_deep_traces.py ffxiv_dx11.6.28h.exe 6.28h-traces + """ opcodes_db, blocks = extract_opcode_data(exe_file) pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True) diff --git a/generate_opcodes_file.py b/generate_opcodes_file.py index 9a36cc9..8b1f5d6 100644 --- a/generate_opcodes_file.py +++ b/generate_opcodes_file.py @@ -49,8 +49,20 @@ def replace_line_with_new_opcode(line, diff, ver): @click.argument("new_version_string") @click.argument("diff_file", type=click.File("r")) @click.argument("opcodes_file", type=click.File("r")) -@click.option("--reverse", is_flag=True) +@click.option( + "--reverse", is_flag=True, help="Applies the diff file in the opposite direction" +) def generate_opcodes_file(new_version_string, diff_file, opcodes_file, reverse): + """ + Parses an OPCODES_FILE and applies a JSON DIFF_FILE to generate a new one. + The opcodes file is basically anything that has syntax resembling Sapphire's + `Ipcs.h`. It doesn't do any C++ header parsing; it's just a bunch of regexes + slapped together but it works. + + Example: + + python generate_opcodes_file.py 6.30h diff.json Ipcs.h + """ diff = load_diff_file(diff_file, reverse) queued_lines = [] for line in opcodes_file.readlines(): diff --git a/minor_patch_diff.py b/minor_patch_diff.py index 162b0f0..8fd874b 100644 --- a/minor_patch_diff.py +++ b/minor_patch_diff.py @@ -190,6 +190,26 @@ def find_opcode_matches(old_opcodes_db, new_opcodes_db): "new_exe", type=click.Path(exists=True, dir_okay=False, resolve_path=True) ) def minor_patch_diff(old_exe, new_exe): + """ + Generates an opcode diff file for minor patches (e.g 6.30 => 6.30h). + + This script outputs to stdout, so pipe it to a json file. + + The format of the output is a list (all fields are optional): + + \b + [ + { + "old": (list of opcodes in the switch case), + "new": (list of opcodes in the switch case), + }, + ... + ] + + Example: + + python minor_patch_diff.py ffxiv_dx11.old.exe ffxiv_dx11.new.exe > diff.json + """ old_opcodes_db = extract_opcode_data(old_exe) new_opcodes_db = extract_opcode_data(new_exe) diff --git a/traces_diff.py b/traces_diff.py index 7f71624..9f6adbd 100644 --- a/traces_diff.py +++ b/traces_diff.py @@ -187,23 +187,6 @@ class OpcodeMatcher: opcodes where the confidence is greater than the given threshold. 2. Old opcodes for which a match could not be confidently found. 3. New opcodes for which a match could not be confidently found. - - The format of the output is a list (all fields are optional): - [ - { - "old": (list of opcodes in the switch case), - "new": (list of opcodes in the switch case), - "score_lead": (confidence above 2nd best match), - "unknown": (true if a match was not made in this case), - "candidates: [ - { - "set": (list of opcodes in switch case), - "score": (candidate score), - } - ... - ] - } - ] """ output = [] @@ -275,6 +258,36 @@ def print_banner(text): ) @click.argument("output_file", type=click.Path(dir_okay=False, resolve_path=True)) def traces_diff(old_traces, new_traces, output_file): + """ + Compares the OLD_TRACES and NEW_TRACES directories generated by the + `generate_deep_traces.py` script. + + Creates a JSON OUTPUT_FILE containing best matches and opcodes with + ambiguous candidates that did not yield a definite match. + + The format of the output is a list (all fields are optional): + + \b + [ + { + "old": (list of opcodes in the switch case), + "new": (list of opcodes in the switch case), + "score_lead": (confidence above 2nd best match), + "unknown": (true if a match was not made in this case), + "candidates: [ + { + "set": (list of opcodes in switch case), + "score": (candidate score), + }, + ... + ] + }, + ... + ] + Example: + + python traces_diff.py old-traces/ new-traces/ diff.json + """ tokens = Tokens() old_trace_data = TraceData.load_data(old_traces, tokens) new_trace_data = TraceData.load_data(new_traces, tokens)