New mods, tagged Pionaarr 2.0 like Browse info, Divide hold and the V2 phase meter: FLAC (flacplay 0.1.0, FLAC tracks play like WAVs, 16/24 bit up to 192 kHz, also over LINK) and exFAT sticks (exfatfs 0.1.0, exFAT sticks with MBR, GPT or none work like FAT32 ones, what the player writes goes onto them too). Cards, docs sections, builder entries, flash shares (exFAT 33 KB, FLAC 11 KB), boot screen shots of 2.0.0, changelog 2.0.0. Browser build with all 19 mods = patcher byte for byte (UPD e4506114, app 8936a1d6, out/test-all-58). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
413 lines
19 KiB
Python
413 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
"""Writes builder/data.js, the data the web builder (builder.html) applies.
|
|
|
|
usage: make_builder_data.py [--repo ../CDJ900NXSVirtual]
|
|
|
|
Our mods (beatjump, cuecolours, hotcuekeys, ledsaver, keynotation, credit) are made by the emulator
|
|
checkout's mods/patch_cdj900nxs.py for every combination the builder offers;
|
|
their bytes do not depend on which of arnoldasr's mods are chosen (checked
|
|
below). For each combination the bytes our mods change in the application
|
|
go into data.js, except what is the player's own firmware:
|
|
|
|
copy the UTILITY item table (hotcuekeys, ledsaver and keynotation extend a
|
|
copy of it)
|
|
is copied from the user's own application in the browser;
|
|
box the HOT KEYS box is the QUANTIZE label's image recoloured with
|
|
HOT KEYS written on it; the browser makes it from the user's image.
|
|
|
|
arnoldasr's mods (credited on the site; advanced_waveforms now ours, based on his), as patch_cdj900nxs.py splits them
|
|
up, are in data.js the same way: per combination of extended_bpm, phasemeter
|
|
and advanced_waveforms the bytes they change, except the asset blob at 0x08FC2D00,
|
|
which starts with a copy of the firmware's resource table: the browser makes
|
|
it from the user's application (the port of patch_cdj900nxs.assets()).
|
|
The SHA-256 of the finished application for every combination is in data.js
|
|
too; the browser checks its result against it.
|
|
|
|
Before data.js is written, the browser's way of applying the data is redone
|
|
here in Python for every combination of arnoldasr's mods and ours and
|
|
compared byte for byte with patch_cdj900nxs.py's result.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
WEB = os.path.dirname(HERE)
|
|
OURS = ('beatjump', 'cuecolours', 'hotcuekeys', 'ledsaver', 'keynotation', 'credit')
|
|
NEEDS = {'hotcuekeys': 'cuecolours', 'beatjump': 'hotcuekeys'}
|
|
UT_TABLE_STOCK = 0x08C6C8DC
|
|
UT_TABLE_LEN = 17 * 52
|
|
BASE = 0x08000000
|
|
ASSETS = None
|
|
|
|
|
|
def sha(b):
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def valid(combo):
|
|
return all(NEEDS[m] in combo for m in combo if m in NEEDS)
|
|
|
|
|
|
def runs(base, final):
|
|
"""[(address, bytes)] where final differs from base (short equal gaps joined)."""
|
|
n = len(final)
|
|
base = base + bytes(n - len(base))
|
|
out, i = [], 0
|
|
while i < n:
|
|
if base[i] != final[i]:
|
|
j = i
|
|
while j < n and (base[j] != final[j] or (j + 16 < n and base[j:j + 16] != final[j:j + 16])):
|
|
j += 1
|
|
out.append((BASE + i, final[i:j]))
|
|
i = j
|
|
else:
|
|
i += 1
|
|
return out
|
|
|
|
|
|
def spans(idx):
|
|
"""[(lo, hi)] of the runs of consecutive numbers in idx (sorted)."""
|
|
out = []
|
|
for i in idx:
|
|
if out and out[-1][1] == i:
|
|
out[-1] = (out[-1][0], i + 1)
|
|
else:
|
|
out.append((i, i + 1))
|
|
return out
|
|
|
|
|
|
def ops_for(stock, final, boxes):
|
|
"""The browser's operations for one combination: data, copy, box."""
|
|
table = stock[UT_TABLE_STOCK - BASE:UT_TABLE_STOCK - BASE + UT_TABLE_LEN]
|
|
out = [dict(box, op='box') for box in boxes] # whole, outside the runs
|
|
masked = bytearray(final)
|
|
padded = stock + bytes(len(final) - len(stock))
|
|
for box in boxes:
|
|
off = box['addr'] - BASE
|
|
masked[off:off + box['len']] = padded[off:off + box['len']]
|
|
for addr, data in runs(stock, bytes(masked)):
|
|
# (start, end, ops) inside this run: a copy of the table, a few of its
|
|
# items changed (beatjump renames SCREEN SAVER), is the copy and then
|
|
# our bytes over it
|
|
cuts = []
|
|
k = data.find(table[:52])
|
|
while k >= 0:
|
|
seg = data[k:k + UT_TABLE_LEN]
|
|
diff = [i for i in range(len(seg)) if seg[i] != table[i]] if len(seg) == UT_TABLE_LEN else None
|
|
if diff is not None and len(diff) <= 2 * 52:
|
|
ops = [{'op': 'copy', 'addr': addr + k, 'src': UT_TABLE_STOCK, 'len': UT_TABLE_LEN}]
|
|
for lo, hi in spans(diff):
|
|
ops.append({'op': 'data', 'addr': addr + k + lo, 'data': seg[lo:hi]})
|
|
cuts.append((k, k + UT_TABLE_LEN, ops))
|
|
k = data.find(table[:52], k + UT_TABLE_LEN)
|
|
else:
|
|
k = data.find(table[:52], k + 1)
|
|
cuts.sort(key=lambda c: c[0])
|
|
pos = 0
|
|
for lo, hi, ops in cuts:
|
|
if lo > pos:
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:lo]})
|
|
out.extend(ops)
|
|
pos = hi
|
|
if pos < len(data):
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:]})
|
|
return out
|
|
|
|
|
|
def apply_ops(app, stock, ops, hk, size):
|
|
"""What builder.js does with the operations (redone here to check them):
|
|
the application grown with zeros to `size` first."""
|
|
out = bytearray(app) + bytes(max(0, size - len(app)))
|
|
for o in ops:
|
|
off = o['addr'] - BASE
|
|
if o['op'] == 'data':
|
|
out[off:off + len(o['data'])] = o['data']
|
|
elif o['op'] == 'copy':
|
|
out[off:off + o['len']] = stock[o['src'] - BASE:o['src'] - BASE + o['len']]
|
|
elif o['op'] == 'assets':
|
|
out[off:off + o['len']] = ASSETS(stock)[0][:o['len']]
|
|
else:
|
|
_, _, px = hk.box_pixels(stock, o['res'], o['bg'], o['ink'], o['text'])
|
|
out[off:off + o['len']] = px
|
|
return bytes(out)
|
|
|
|
|
|
MODS_ALL = ('extended_bpm', 'phasemeter', 'advanced_waveforms', 'beatjump', 'cuecolours', 'hotcuekeys',
|
|
'ledsaver', 'keynotation', 'usbautoload', 'quicksearch', 'cuenames', 'played', 'endwarn', 'wide_waveform',
|
|
'browseinfo', 'divhold', 'flacplay', 'exfatfs', 'credit')
|
|
FLASH_NEEDS = {'beatjump': ('cuecolours', 'hotcuekeys'), 'hotcuekeys': ('cuecolours',), 'cuenames': ('cuecolours',)}
|
|
FLASH_CACHE = os.path.join(HERE, 'flash_sizes.json')
|
|
|
|
|
|
def _packed_len(job):
|
|
"""(repo, names) -> the LZSS stream's length of the application with those mods (one process each)."""
|
|
repo, names = job
|
|
sys.path[:0] = [os.path.join(repo, 'mods'), os.path.join(repo, 'mods', 'cdj900nxs'),
|
|
os.path.join(repo, 'scripts', 'firmware')]
|
|
import patch_cdj900nxs as p
|
|
import lzss_encode
|
|
with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
app = p.patch(stock, list(names)) if names else stock
|
|
return names, sha(app), len(lzss_encode.pack(app))
|
|
|
|
|
|
def flash_sizes(repo, p):
|
|
"""What each mod takes of the player's flash: the application is stored LZSS-packed in
|
|
0x40000-0x7E0000 (upd.APP_END - APP_START bytes). A mod's share is the packed size with it
|
|
(and what it needs) minus without it; the sum over a choice is an estimate (the packer finds
|
|
a little more or less to share), the exact number the build tells. Packing takes a minute a
|
|
run, so results are cached by the application's SHA-256 (tools/flash_sizes.json)."""
|
|
sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher'))
|
|
import upd
|
|
from multiprocessing import Pool
|
|
try:
|
|
with open(FLASH_CACHE, encoding='utf-8') as f:
|
|
cache = json.load(f)
|
|
except (OSError, ValueError):
|
|
cache = {}
|
|
with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
def removed(m): # m and every mod that needs it, directly or not
|
|
out, grow = {m}, True
|
|
while grow:
|
|
grow = False
|
|
for n, d in FLASH_NEEDS.items():
|
|
if n not in out and out & set(d):
|
|
out.add(n)
|
|
grow = True
|
|
return out
|
|
jobs = {(): None, MODS_ALL: None}
|
|
for m in MODS_ALL:
|
|
jobs[tuple(n for n in MODS_ALL if n not in removed(m))] = None
|
|
d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ()))
|
|
jobs[d] = None
|
|
jobs[tuple(n for n in MODS_ALL if n in d + (m,))] = None
|
|
todo, known = [], {}
|
|
for names in jobs:
|
|
app_sha = sha(p.patch(stock, list(names)) if names else stock)
|
|
if app_sha in cache:
|
|
known[names] = cache[app_sha]
|
|
else:
|
|
todo.append((repo, names))
|
|
if todo:
|
|
print('packing %d applications for the flash sizes (about a minute each, in parallel)' % len(todo))
|
|
with Pool(min(len(todo), max(1, (os.cpu_count() or 2) - 1))) as pool:
|
|
for names, app_sha, n in pool.imap_unordered(_packed_len, todo):
|
|
cache[app_sha] = n
|
|
known[names] = n
|
|
print(' %-60s %d' % (','.join(names) or 'stock', n))
|
|
with open(FLASH_CACHE, 'w', encoding='utf-8', newline=chr(10)) as f:
|
|
json.dump(cache, f, indent=1, sort_keys=True)
|
|
region = lambda n: len(upd.pack_region(bytes(n)))
|
|
# The packer shares a lot between the mods (alone each costs 7-30 KB, all together 50 KB), so a
|
|
# sum of single sizes is far off. What every mod brings anyway (the frame: the smallest mod alone,
|
|
# the boot text, costs it) is counted once; the rest is split by each mod's marginal size in the
|
|
# full build (all mods packed minus all but it, and what needs it). Exact with none and with all.
|
|
alone = {}
|
|
for m in MODS_ALL:
|
|
d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ()))
|
|
alone[m] = known[tuple(n for n in MODS_ALL if n in d + (m,))] - known[d]
|
|
frame = min(v for m, v in alone.items() if m not in FLASH_NEEDS) # (one that needs others adds to them)
|
|
marg = {}
|
|
for m in sorted(MODS_ALL, key=lambda m: len(removed(m))):
|
|
rest = tuple(n for n in MODS_ALL if n not in removed(m))
|
|
marg[m] = max(0, known[MODS_ALL] - known[rest] - sum(marg[n] for n in removed(m) - {m}))
|
|
total = region(known[MODS_ALL]) - region(known[()]) - frame
|
|
k = total / float(sum(marg.values()) or 1)
|
|
mods = {m: max(1, int(round(marg[m] * k))) for m in MODS_ALL}
|
|
out = {'size': upd.APP_END - upd.APP_START, 'stock': region(known[()]),
|
|
'all': region(known[MODS_ALL]), 'common': frame, 'mods': mods}
|
|
print('flash: %d of %d with no mods, %d with all; shares %d + frame %d'
|
|
% (out['stock'], out['size'], out['all'], sum(mods.values()), out['common']))
|
|
print(' ' + ', '.join('%s %d' % kv for kv in sorted(mods.items(), key=lambda kv: -kv[1])))
|
|
return out
|
|
|
|
|
|
def python_bundle(repo):
|
|
"""Our patch code for the browser: mods/patch_cdj900nxs.py and every module
|
|
in mods/cdj900nxs (not arnoldasr's checkout, not the tests)."""
|
|
files = {'mods/patch_cdj900nxs.py': os.path.join(repo, 'mods', 'patch_cdj900nxs.py')}
|
|
d = os.path.join(repo, 'mods', 'cdj900nxs')
|
|
for n in sorted(os.listdir(d)):
|
|
if n.endswith('.py'):
|
|
files['mods/cdj900nxs/' + n] = os.path.join(d, n)
|
|
out = {}
|
|
for rel, path in files.items():
|
|
with open(path, encoding='utf-8') as f:
|
|
out[rel] = f.read()
|
|
return out
|
|
|
|
|
|
def set_release(repo):
|
|
"""The boot screen (the credit mod) names the Pionaarr Mods release: always
|
|
the site's version (tools/changelog.json), written into credit.py's
|
|
RELEASE before anything is built."""
|
|
import re
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
site = json.load(f)['site'][0]['version']
|
|
path = os.path.join(repo, 'mods', 'cdj900nxs', 'credit.py')
|
|
with open(path, encoding='utf-8') as f:
|
|
src = f.read()
|
|
new = re.sub(r"^RELEASE = '[^']*'", "RELEASE = '%s'" % site, src, count=1, flags=re.M)
|
|
if new != src:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('set credit.RELEASE to %s (the boot screen)' % site)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--repo', default=os.path.join(os.path.dirname(WEB), 'CDJ900NXSVirtual'))
|
|
args = ap.parse_args()
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods'))
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods', 'cdj900nxs'))
|
|
set_release(args.repo)
|
|
import patch_cdj900nxs as p
|
|
import hotcuekeys as hk
|
|
import cuecolours
|
|
with open(os.path.join(args.repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
if sha(stock) != p.APP_SHA256:
|
|
raise SystemExit('not the 1.31 application')
|
|
|
|
global ASSETS
|
|
ASSETS = p.assets
|
|
_, labels = hk.routine(hk.DECODER, cuecolours.HOT_TABLE)
|
|
w, h, on = hk.box_pixels(stock, hk.RES_QUANTIZE, 0xF800, 0x0000)
|
|
boxes = [{'addr': labels['PIX_ON'], 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xF800, 'ink': 0x0000,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': labels['PIX_OFF'], 'len': len(on), 'res': hk.RES_QUANTIZE_OFF, 'bg': 0x0000, 'ink': 0x4208,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': hk.PIX_LOAD, 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xFC60, 'ink': 0x0000,
|
|
'text': 'LOADING'}]
|
|
|
|
combos = [c for n in range(1, len(OURS) + 1) for c in itertools.combinations(OURS, n) if valid(c)]
|
|
variants, chunks = {}, {}
|
|
|
|
def encode(ops):
|
|
enc = []
|
|
for o in ops:
|
|
if o['op'] == 'data':
|
|
key = sha(o['data'])[:16]
|
|
chunks[key] = base64.b64encode(o['data']).decode()
|
|
enc.append(['data', o['addr'], key])
|
|
elif o['op'] == 'copy':
|
|
enc.append(['copy', o['addr'], o['src'], o['len']])
|
|
elif o['op'] == 'assets':
|
|
enc.append(['assets', o['addr'], o['len']])
|
|
else:
|
|
enc.append(['box', o['addr'], o['res'], o['bg'], o['ink'], o['text']])
|
|
return enc
|
|
|
|
# arnoldasr's mods, split up: per combination their bytes, but the asset
|
|
# blob (it starts with a copy of the firmware's resource table) made by
|
|
# the browser from the user's own application
|
|
upstream, up_apps = {}, {(): stock}
|
|
for k in range(1, len(p.UPSTREAM) + 1):
|
|
for up in itertools.combinations(p.UPSTREAM, k):
|
|
app = p.patch_upstream(stock, list(up))
|
|
# the asset blob itself (code of ours after it, advanced_waveforms'
|
|
# fix, goes as data)
|
|
blob_len = min(len(ASSETS(stock)[0]), len(app) - (p.DATA_BASE - BASE))
|
|
masked = bytearray(app)
|
|
masked[p.DATA_BASE - BASE:p.DATA_BASE - BASE + blob_len] = bytes(blob_len)
|
|
ops = ops_for(stock, bytes(masked), [])
|
|
ops.append({'op': 'assets', 'addr': p.DATA_BASE, 'len': blob_len})
|
|
if apply_ops(stock, stock, ops, hk, len(app)) != app:
|
|
raise SystemExit('%s: the operations do not give the patcher result' % ','.join(up))
|
|
upstream[','.join(up)] = {'size': len(app), 'ops': encode(ops)}
|
|
up_apps[up] = app
|
|
print('%-40s %3d operations' % (','.join(up), len(ops)))
|
|
# stage 1 only: the browser checks its result against these; stage 2 (ours)
|
|
# runs there as Python, the patcher's own code (builder/py/bundle.json)
|
|
expect = {','.join(up): sha(app) for up, app in up_apps.items() if up}
|
|
bundle = python_bundle(args.repo)
|
|
with open(os.path.join(WEB, 'builder', 'py', 'bundle.json'), 'w', encoding='utf-8', newline='\n') as f:
|
|
json.dump(bundle, f, separators=(',', ':'))
|
|
# stage 2 as the browser runs it: patch_ours on stage 1's application
|
|
for names in (['credit'], list(MODS_ALL), ['phasemeter'], ['advanced_waveforms', 'cuecolours', 'cuenames']):
|
|
up = tuple(n for n in p.UPSTREAM if n in names)
|
|
if p.patch_ours(up_apps[up], stock, p.check_names(names)) != p.patch(stock, names):
|
|
raise SystemExit('%s: stage 1 + stage 2 differs from the patcher' % ','.join(names))
|
|
print('checked: stage 1 data + stage 2 Python give the patcher bytes; %d Python files' % len(bundle))
|
|
|
|
glyphs = {c: rows for c, rows in hk.GLYPHS.items()}
|
|
data = {
|
|
'stockUpdSha256': '', # filled from the patcher's upd.py below
|
|
'stockAppSha256': p.APP_SHA256,
|
|
'appSize': p.APP_SIZE,
|
|
'dataBase': p.DATA_BASE,
|
|
'upstream': upstream,
|
|
'expect': expect,
|
|
'chunks': chunks,
|
|
}
|
|
sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher'))
|
|
import upd
|
|
data['stockUpdSha256'] = upd.STOCK_UPD_SHA256
|
|
data['flash'] = flash_sizes(args.repo, p)
|
|
for key, b64 in chunks.items(): # no firmware in what we ship
|
|
blob = base64.b64decode(b64)
|
|
for i in range(0, max(0, len(blob) - 47), 8):
|
|
win = blob[i:i + 48]
|
|
if len(set(win)) > 2 and stock.find(win) >= 0:
|
|
raise SystemExit('chunk %s holds firmware bytes (at %d)' % (key, i))
|
|
out = os.path.join(WEB, 'builder', 'data.js')
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write('// Generated by tools/make_builder_data.py -- do not edit.\n')
|
|
f.write('window.BUILDER_DATA = ')
|
|
json.dump(data, f, separators=(',', ':'))
|
|
f.write(';\n')
|
|
print('wrote %s (%d bytes, %d variants, %d chunks)' % (out, os.path.getsize(out), len(upstream), len(chunks)))
|
|
write_versions(p)
|
|
|
|
|
|
def write_versions(p):
|
|
"""assets/versions.js: the site's version and changelog (tools/changelog.json)
|
|
and every mod's version as patch_cdj900nxs.VERSIONS has it."""
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
log = json.load(f)
|
|
data = {'site': log['site'][0]['version'], 'date': log['site'][0]['date'], 'log': log['site'],
|
|
'mods': dict(p.VERSIONS), 'modlog': log.get('mods', {})}
|
|
out = os.path.join(WEB, 'assets', 'versions.js')
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write("// Generated by tools/make_builder_data.py from tools/changelog.json and the mods' versions.\n")
|
|
f.write('window.CDJ_VERSIONS = ' + json.dumps(data, indent=1, ensure_ascii=False) + ';\n')
|
|
print('wrote %s (site %s)' % (out, data['site']))
|
|
# the pages load the site's own CSS and scripts with ?v=<site version>, so
|
|
# a browser takes the new ones after an update
|
|
import re
|
|
for name in sorted(os.listdir(WEB)):
|
|
if not name.endswith('.html'):
|
|
continue
|
|
path = os.path.join(WEB, name)
|
|
with open(path, encoding='utf-8') as f:
|
|
html = f.read()
|
|
new = re.sub(r'(href|src)="((?:style\.css|script\.js|assets/versions\.js|builder/[a-z]+\.js))(\?v=[^"]*)?"',
|
|
lambda m: '%s="%s?v=%s"' % (m.group(1), m.group(2), data['site']), html)
|
|
new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], new) # the boot text
|
|
if new != html:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped %s with ?v=%s' % (name, data['site']))
|
|
ui = os.path.join(WEB, 'builder', 'ui.js')
|
|
with open(ui, encoding='utf-8') as f:
|
|
js = f.read()
|
|
new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], js)
|
|
if new != js:
|
|
with open(ui, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped builder/ui.js with Pionaarr Mods %s' % data['site'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|