diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..22fc911 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +out/ +*.UPD +__pycache__/ +.pytest_cache/ diff --git a/README.md b/README.md index e69de29..25e4ba7 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,72 @@ +# CDJ-900NXS firmware patcher + +Builds a new `C900NXS.UPD` from the official CDJ-900NXS firmware 1.31 with +the mods you choose, for the player's own USB update. + +```sh +python cdj900nxs_patcher.py C900NXS.UPD # choose in a list +python cdj900nxs_patcher.py C900NXS.UPD --mods beatjump,cuecolours,hotcuekeys +python cdj900nxs_patcher.py C900NXS.UPD --stock # the original, to go back +python cdj900nxs_patcher.py --list +``` + +In the list: **arrow keys** move, **space** ticks or unticks, **a** ticks all +or none, **enter** builds, **q** quits. Terminals that cannot pass single +keys (Git Bash's mintty, a pipe) get a numbered list instead. The result goes +to `out/-/C900NXS.UPD` with a `report.txt` of every check. + +## What it guarantees + +- **Only the original file is accepted**: SHA-256 + `e96bca7d1ac983985992a8e5fa7813e1ebf04bef514e0c6cfd0a8375f667f7f9`, and the + 1.31 application inside it is checked too. +- **Only the main application is replaced.** The boot loader and recovery + loader (MAIN's records below flash 0x40000), MAIN's S0 header and S7 entry + records, and the DRIV and PANL sections are copied from the original byte + for byte. Before anything is written the finished file is read back and + checked: every section CRC and S-record checksum; DRIV and PANL equal to the + original; MAIN's header equal but for its version label, application-only + mode kept; every boot loader and recovery record and byte equal; no record + past 0x7E0000; the packed application's checksum; and the application, + unpacked the way the boot loader does it, equal to the patched one. +- **Labelled 1.32**, so that the player's updater does not skip the update as + the version it already has. The application still reports 1.31 (as + arnoldasr's builds do). `--stock` gives the original firmware with only that + label changed: three bytes differ from the official file (the digit and the + section CRC), exactly as arnoldasr's back-to-stock file, which installed on + his test player. + +Checked in the emulator: both the `--stock` file and one with every mod were +written into an emulated flash and booted through Pioneer's own boot loader, +which unpacked and started them. + +## Mods + +From the emulator checkout `../CDJ900NXSVirtual` (`--repo` for another place): +`mods/patch_cdj900nxs.py` and the mods next to it. + +| mod | what it does | +|---|---| +| `code` | arnoldasr: BPM with two decimals, two-line phase meter, RGB waveform | +| `utility` | arnoldasr: UTILITY > PHASE METER switch (needs `code`) | +| `overview` | arnoldasr: overview waveform colour gradient | +| `beatjump` | TRACK SEARCH jumps 1/2/4/8 beats, UTILITY > BEAT JUMP | +| `cuecolours` | memory cues in their rekordbox colours, hot cues shown | +| `hotcuekeys` | UTILITY > HOT CUE KEYS; QUANTIZE short = hot key mode | + +arnoldasr's mods need his repository checked out in +`../CDJ900NXSVirtual/mods/cdj900nxs/upstream` (it has no licence, so it is +not copied here); without it they are greyed out. + +## Risk + +Nothing here talks to a player; you copy the file to a FAT32 USB stick and run +the update yourself. Our mods have only run in the emulator. Recovery from an +application that crashes at start-up has not been tested by anyone. Keep the +`--stock` file at hand. + +## Tests + +```sh +CDJ900NXS_UPD=/path/to/C900NXS.UPD python -m pytest test_patcher.py +``` diff --git a/cdj900nxs_patcher.py b/cdj900nxs_patcher.py new file mode 100644 index 0000000..dcf87e1 --- /dev/null +++ b/cdj900nxs_patcher.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-or-later +"""CDJ-900NXS firmware patcher: the official 1.31 update with the mods you +choose, as a new C900NXS.UPD for the player's own USB update. + + python cdj900nxs_patcher.py C900NXS.UPD # choose in a list, write out/... + python cdj900nxs_patcher.py C900NXS.UPD --mods beatjump,cuecolours + python cdj900nxs_patcher.py C900NXS.UPD --stock # unmodified, to go back + python cdj900nxs_patcher.py --list + + * Only the original update is accepted (its SHA-256 is checked). + * Only the main application is replaced. The boot loader and recovery + loader (flash 0x0-0x3FFFF), the S-record header and entry, and the DRIV + and PANL sections are copied from the original byte for byte, and checked + again on the finished file before it is written. + * The new file says 1.32 in its MAIN header so that the player's updater + does not skip it as the version it already has; the application itself + still reports 1.31 (as arnoldasr's builds do). Only that label differs. + * Nothing here talks to a player. You copy the file to a USB stick and run + the update on the player yourself, at your own risk: nothing of this has + been tested on hardware except the relabelled stock update (by arnoldasr), + and recovery from an application that crashes at start-up is untested. + +The mods come from the CDJ900NXSVirtual checkout next to this folder +(--repo): mods/patch_cdj900nxs.py, which also needs arnoldasr's checkout in +mods/cdj900nxs/upstream for his mods (code, utility, overview). +""" +import argparse +import os +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import upd # noqa: E402 + +DEFAULT_REPO = os.path.join(os.path.dirname(HERE), 'CDJ900NXSVirtual') +UPSTREAM_MODS = ('code', 'utility', 'overview') +NEEDS = {'utility': 'code'} # the PHASE METER switch chooses the code mod's meter +LABEL = '1.32' + + +def load_repo(repo): + """patch_cdj900nxs and the LZSS packer from the emulator checkout.""" + mods_dir = os.path.join(repo, 'mods') + fw_dir = os.path.join(repo, 'scripts', 'firmware') + if not os.path.isfile(os.path.join(mods_dir, 'patch_cdj900nxs.py')): + raise SystemExit('error: no mods/patch_cdj900nxs.py in %s (use --repo)' % repo) + for d in (mods_dir, fw_dir): + if d not in sys.path: + sys.path.insert(0, d) + import patch_cdj900nxs + import lzss_encode + return patch_cdj900nxs, lzss_encode + + +def available(patcher, repo): + """(name, description, usable, why not) for every mod.""" + upstream = os.path.isdir(os.environ.get('CDJ900NXS_MODS_SRC') + or os.path.join(repo, 'mods', 'cdj900nxs', 'upstream')) + out = [] + for name, what in patcher.MODS.items(): + ok = upstream or name not in UPSTREAM_MODS + out.append((name, what, ok, '' if ok else "needs arnoldasr's checkout in mods/cdj900nxs/upstream")) + return out + + +# --- choosing ------------------------------------------------------------------ + +def _keys(): + """A function returning 'up', 'down', 'space', 'enter', 'all', 'quit' per + key press, or None when the terminal cannot be read key by key.""" + if not sys.stdin.isatty(): + return None + if os.name == 'nt': + import msvcrt + + def read(): + c = msvcrt.getwch() + if c in ('\x00', '\xe0'): + return {'H': 'up', 'P': 'down'}.get(msvcrt.getwch()) + return {' ': 'space', '\r': 'enter', 'a': 'all', 'q': 'quit', '\x1b': 'quit', + '\x03': 'quit'}.get(c.lower()) + return read + import termios + import tty + + def read(): + fd = sys.stdin.fileno() + old = termios.tcgetattr(fd) + try: + tty.setraw(fd) + c = sys.stdin.read(1) + if c == '\x1b': + if sys.stdin.read(1) == '[': + return {'A': 'up', 'B': 'down'}.get(sys.stdin.read(1)) + return 'quit' + finally: + termios.tcsetattr(fd, termios.TCSADRAIN, old) + return {' ': 'space', '\r': 'enter', '\n': 'enter', 'a': 'all', 'q': 'quit', + '\x03': 'quit'}.get(c.lower()) + return read + + +def _fix_needs(chosen, mods): + for name, need in NEEDS.items(): + if name in chosen and need not in chosen and any(m[0] == need and m[2] for m in mods): + chosen.add(need) + + +def choose(mods): + """The interactive list: arrows move, space ticks, enter goes on.""" + read = _keys() + if read is None: + return choose_by_number(mods) + if os.name == 'nt': + os.system('') # VT sequences on in the Windows console + chosen = {m[0] for m in mods if m[2]} + at = 0 + lines = len(mods) + 3 + + def draw(first): + if not first: + sys.stdout.write('\x1b[%dA' % lines) + sys.stdout.write('\x1b[2K Mods (arrows move, space ticks, a = all/none, enter goes on, q quits)\n') + for i, (name, what, ok, why) in enumerate(mods): + mark = '[x]' if name in chosen else ('[ ]' if ok else ' - ') + cursor = '>' if i == at else ' ' + text = '%s %s %-11s %s' % (cursor, mark, name, what if ok else why) + sys.stdout.write('\x1b[2K' + ('\x1b[7m%s\x1b[0m' % text if i == at else text)[:200] + '\n') + need = ', '.join('%s needs %s' % kv for kv in NEEDS.items()) + sys.stdout.write('\x1b[2K (%s)\n\x1b[2K\n' % need) + sys.stdout.flush() + + draw(True) + while True: + k = read() + if k == 'up': + at = (at - 1) % len(mods) + elif k == 'down': + at = (at + 1) % len(mods) + elif k == 'space' and mods[at][2]: + chosen ^= {mods[at][0]} + _fix_needs(chosen, mods) + elif k == 'all': + usable = {m[0] for m in mods if m[2]} + chosen = set() if chosen == usable else usable + elif k == 'enter': + break + elif k == 'quit': + raise SystemExit('nothing written') + draw(False) + return [m[0] for m in mods if m[0] in chosen] + + +def choose_by_number(mods): + """For terminals that cannot be read key by key (mintty, pipes).""" + chosen = {m[0] for m in mods if m[2]} + while True: + print('\n Mods (all usable ones are ticked):') + for i, (name, what, ok, why) in enumerate(mods, 1): + mark = '[x]' if name in chosen else ('[ ]' if ok else ' - ') + print(' %d %s %-11s %s' % (i, mark, name, what if ok else why)) + answer = input(' Numbers to tick/untick (e.g. 1 3), enter to go on, q to quit: ').strip() + if not answer: + break + if answer.lower() == 'q': + raise SystemExit('nothing written') + for tok in answer.replace(',', ' ').split(): + if tok.isdigit() and 1 <= int(tok) <= len(mods) and mods[int(tok) - 1][2]: + chosen ^= {mods[int(tok) - 1][0]} + _fix_needs(chosen, mods) + return [m[0] for m in mods if m[0] in chosen] + + +# --- building ------------------------------------------------------------------ + +def build(stock, names, patcher, lzss, label=LABEL, log=print): + """(update bytes, application, list of checks).""" + if names: + try: + app = patcher.patch(stock.application, names) + except SystemExit as e: + raise upd.UpdError(str(e)) + log('packing the application (LZSS, about a minute and a half)...') + t = time.time() + stream = lzss.pack(app) + log(' %d -> %d bytes in %.0f s' % (len(app), len(stream), time.time() - t)) + else: + import struct + app = stock.application + size = struct.unpack_from(' size: + raise UpdError('%s: a record reaches past 0x%X' % (self.name, size)) + image[r.address:r.address + len(r.data)] = r.data + return bytes(image) + + +def encode_section(header, records): + body = header + b''.join(r.encode() for r in records) + return body + struct.pack('= n: + break + if flags >> bit & 1: + b = stream[pos] + pos += 1 + out.append(b) + ring[r] = b + r = (r + 1) & 0xFFF + else: + if pos + 2 > n: + raise UpdError('the packed application ends inside a match') + lo, hi = stream[pos], stream[pos + 1] + pos += 2 + at, count = lo | (hi & 0xF0) << 4, (hi & 0x0F) + 3 + for k in range(count): + b = ring[(at + k) & 0xFFF] + out.append(b) + ring[r] = b + r = (r + 1) & 0xFFF + return bytes(out) + + +def region(flash): + """(application, packed region bytes) from the flash image.""" + size = struct.unpack_from(' APP_END: + raise UpdError('the packed application runs past 0x%X' % APP_END) + stored = struct.unpack_from(' APP_START for r in low): + raise UpdError('a stock record crosses into the application') + return low + + +def relabel(header, version): + old = HEADER.match(header).group(0) + new = old[:-4] + version.encode() + if not re.fullmatch(r'\d\.\d\d', version): + raise UpdError('the version label must look like 1.32') + out = header.replace(old, new, 1) + if out[:len(old) - 4] != header[:len(old) - 4] or out[len(old):] != header[len(old):]: + raise UpdError('relabelling would change more than the version') + return out + + +def build(stock, application, packed_stream, label='1.32'): + """The new update: stock DRIV and PANL, MAIN with the stock header (version + label only changed), the stock S0, the stock boot loader and recovery + records, the new application region, and the stock S7 end record.""" + main = stock.sections[1] + reg = pack_region(packed_stream) + if APP_START + len(reg) > APP_END: + raise UpdError('the packed application needs %d bytes, the flash has %d' + % (len(reg), APP_END - APP_START)) + records = [main.records[0]] + stock.boot_records() + records += [Record(2, APP_START + off, reg[off:off + RECORD_DATA]) for off in range(0, len(reg), RECORD_DATA)] + records.append(main.records[-1]) + blob = encode_section(relabel(main.header, label) if label else main.header, records) + return join([stock.sections[0].blob, blob, stock.sections[2].blob]) + + +def verify(raw, stock, application, label='1.32'): + """Every check on a finished file; returns the list of what was checked.""" + done = [] + sections = parse(raw) + done.append('section lengths, CRC-16 of every section, every S-record checksum') + if sections[0].blob != stock.sections[0].blob or sections[2].blob != stock.sections[2].blob: + raise UpdError('DRIV or PANL differs from stock') + done.append('DRIV and PANL byte for byte as stock') + main, smain = sections[1], stock.sections[1] + if main.header != (relabel(smain.header, label) if label else smain.header): + raise UpdError('the MAIN header differs beyond its version label') + if main.header[31:32] != b'0': + raise UpdError('the MAIN header no longer selects application-only mode') + done.append('MAIN header as stock but for the label %s; application-only mode kept' % (label or '1.31')) + if main.records[0] != smain.records[0] or main.records[-1] != smain.records[-1]: + raise UpdError('the S0 header or S7 entry record changed') + done.append('S0 header record and S7 entry record as stock') + low = [r for r in main.data_records() if r.address < APP_START] + if low != stock.boot_records(): + raise UpdError('the boot loader or recovery records changed') + flash = main.flash() + if flash[:APP_START] != stock.flash[:APP_START]: + raise UpdError('the boot loader or recovery bytes changed') + done.append('boot loader and recovery (flash 0x0-0x3FFFF): every record and byte as stock') + if any(r.address < APP_START < r.address + len(r.data) for r in main.data_records()): + raise UpdError('a record crosses into the boot area') + done.append('no record reaches past 0x%X or into the boot area' % APP_END) + app, reg = region(flash) + done.append('packed application: additive checksum') + if app != application: + raise UpdError('the update does not unpack to the patched application') + done.append('unpacked with the boot loader\'s LZSS: exactly the patched application (SHA-256 %s)' + % sha256(app)[:16]) + done.append('flash used by the application: %d of %d bytes' % (len(reg), APP_END - APP_START)) + return done