Thiemo TizianiandClaude Opus 5.5
5537afb53d
make_builder_data.py writes the site version into the credit mod's RELEASE and into the pages' and the builder's boot text on every run; new boot screen shots. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
324 lines
14 KiB
Python
324 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
"""Writes builder/data.js, the data the web builder (builder.html) applies.
|
|
|
|
usage: make_builder_data.py [--repo ../CDJ900NXSVirtual]
|
|
|
|
Our mods (beatjump, cuecolours, hotcuekeys, ledsaver, keynotation, credit) are made by the emulator
|
|
checkout's mods/patch_cdj900nxs.py for every combination the builder offers;
|
|
their bytes do not depend on which of arnoldasr's mods are chosen (checked
|
|
below). For each combination the bytes our mods change in the application
|
|
go into data.js, except what is the player's own firmware:
|
|
|
|
copy the UTILITY item table (hotcuekeys, ledsaver and keynotation extend a
|
|
copy of it)
|
|
is copied from the user's own application in the browser;
|
|
box the HOT KEYS box is the QUANTIZE label's image recoloured with
|
|
HOT KEYS written on it; the browser makes it from the user's image.
|
|
|
|
arnoldasr's mods (credited on the site; advanced_waveforms now ours, based on his), as patch_cdj900nxs.py splits them
|
|
up, are in data.js the same way: per combination of extended_bpm, phasemeter
|
|
and advanced_waveforms the bytes they change, except the asset blob at 0x08FC2D00,
|
|
which starts with a copy of the firmware's resource table: the browser makes
|
|
it from the user's application (the port of patch_cdj900nxs.assets()).
|
|
The SHA-256 of the finished application for every combination is in data.js
|
|
too; the browser checks its result against it.
|
|
|
|
Before data.js is written, the browser's way of applying the data is redone
|
|
here in Python for every combination of arnoldasr's mods and ours and
|
|
compared byte for byte with patch_cdj900nxs.py's result.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
WEB = os.path.dirname(HERE)
|
|
OURS = ('beatjump', 'cuecolours', 'hotcuekeys', 'ledsaver', 'keynotation', 'credit')
|
|
NEEDS = {'hotcuekeys': 'cuecolours', 'beatjump': 'hotcuekeys'}
|
|
UT_TABLE_STOCK = 0x08C6C8DC
|
|
UT_TABLE_LEN = 17 * 52
|
|
BASE = 0x08000000
|
|
ASSETS = None
|
|
|
|
|
|
def sha(b):
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def valid(combo):
|
|
return all(NEEDS[m] in combo for m in combo if m in NEEDS)
|
|
|
|
|
|
def runs(base, final):
|
|
"""[(address, bytes)] where final differs from base (short equal gaps joined)."""
|
|
n = len(final)
|
|
base = base + bytes(n - len(base))
|
|
out, i = [], 0
|
|
while i < n:
|
|
if base[i] != final[i]:
|
|
j = i
|
|
while j < n and (base[j] != final[j] or (j + 16 < n and base[j:j + 16] != final[j:j + 16])):
|
|
j += 1
|
|
out.append((BASE + i, final[i:j]))
|
|
i = j
|
|
else:
|
|
i += 1
|
|
return out
|
|
|
|
|
|
def spans(idx):
|
|
"""[(lo, hi)] of the runs of consecutive numbers in idx (sorted)."""
|
|
out = []
|
|
for i in idx:
|
|
if out and out[-1][1] == i:
|
|
out[-1] = (out[-1][0], i + 1)
|
|
else:
|
|
out.append((i, i + 1))
|
|
return out
|
|
|
|
|
|
def ops_for(stock, final, boxes):
|
|
"""The browser's operations for one combination: data, copy, box."""
|
|
table = stock[UT_TABLE_STOCK - BASE:UT_TABLE_STOCK - BASE + UT_TABLE_LEN]
|
|
out = [dict(box, op='box') for box in boxes] # whole, outside the runs
|
|
masked = bytearray(final)
|
|
padded = stock + bytes(len(final) - len(stock))
|
|
for box in boxes:
|
|
off = box['addr'] - BASE
|
|
masked[off:off + box['len']] = padded[off:off + box['len']]
|
|
for addr, data in runs(stock, bytes(masked)):
|
|
# (start, end, ops) inside this run: a copy of the table, a few of its
|
|
# items changed (beatjump renames SCREEN SAVER), is the copy and then
|
|
# our bytes over it
|
|
cuts = []
|
|
k = data.find(table[:52])
|
|
while k >= 0:
|
|
seg = data[k:k + UT_TABLE_LEN]
|
|
diff = [i for i in range(len(seg)) if seg[i] != table[i]] if len(seg) == UT_TABLE_LEN else None
|
|
if diff is not None and len(diff) <= 2 * 52:
|
|
ops = [{'op': 'copy', 'addr': addr + k, 'src': UT_TABLE_STOCK, 'len': UT_TABLE_LEN}]
|
|
for lo, hi in spans(diff):
|
|
ops.append({'op': 'data', 'addr': addr + k + lo, 'data': seg[lo:hi]})
|
|
cuts.append((k, k + UT_TABLE_LEN, ops))
|
|
k = data.find(table[:52], k + UT_TABLE_LEN)
|
|
else:
|
|
k = data.find(table[:52], k + 1)
|
|
cuts.sort(key=lambda c: c[0])
|
|
pos = 0
|
|
for lo, hi, ops in cuts:
|
|
if lo > pos:
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:lo]})
|
|
out.extend(ops)
|
|
pos = hi
|
|
if pos < len(data):
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:]})
|
|
return out
|
|
|
|
|
|
def apply_ops(app, stock, ops, hk, size):
|
|
"""What builder.js does with the operations (redone here to check them):
|
|
the application grown with zeros to `size` first."""
|
|
out = bytearray(app) + bytes(max(0, size - len(app)))
|
|
for o in ops:
|
|
off = o['addr'] - BASE
|
|
if o['op'] == 'data':
|
|
out[off:off + len(o['data'])] = o['data']
|
|
elif o['op'] == 'copy':
|
|
out[off:off + o['len']] = stock[o['src'] - BASE:o['src'] - BASE + o['len']]
|
|
elif o['op'] == 'assets':
|
|
out[off:off + o['len']] = ASSETS(stock)[0][:o['len']]
|
|
else:
|
|
_, _, px = hk.box_pixels(stock, o['res'], o['bg'], o['ink'], o['text'])
|
|
out[off:off + o['len']] = px
|
|
return bytes(out)
|
|
|
|
|
|
MODS_ALL = ('extended_bpm', 'phasemeter', 'advanced_waveforms', 'beatjump', 'cuecolours', 'hotcuekeys',
|
|
'ledsaver', 'keynotation', 'usbautoload', 'quicksearch', 'cuenames', 'played', 'endwarn', 'credit')
|
|
|
|
|
|
def python_bundle(repo):
|
|
"""Our patch code for the browser: mods/patch_cdj900nxs.py and every module
|
|
in mods/cdj900nxs (not arnoldasr's checkout, not the tests)."""
|
|
files = {'mods/patch_cdj900nxs.py': os.path.join(repo, 'mods', 'patch_cdj900nxs.py')}
|
|
d = os.path.join(repo, 'mods', 'cdj900nxs')
|
|
for n in sorted(os.listdir(d)):
|
|
if n.endswith('.py'):
|
|
files['mods/cdj900nxs/' + n] = os.path.join(d, n)
|
|
out = {}
|
|
for rel, path in files.items():
|
|
with open(path, encoding='utf-8') as f:
|
|
out[rel] = f.read()
|
|
return out
|
|
|
|
|
|
def set_release(repo):
|
|
"""The boot screen (the credit mod) names the Pionaarr Mods release: always
|
|
the site's version (tools/changelog.json), written into credit.py's
|
|
RELEASE before anything is built."""
|
|
import re
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
site = json.load(f)['site'][0]['version']
|
|
path = os.path.join(repo, 'mods', 'cdj900nxs', 'credit.py')
|
|
with open(path, encoding='utf-8') as f:
|
|
src = f.read()
|
|
new = re.sub(r"^RELEASE = '[^']*'", "RELEASE = '%s'" % site, src, count=1, flags=re.M)
|
|
if new != src:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('set credit.RELEASE to %s (the boot screen)' % site)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--repo', default=os.path.join(os.path.dirname(WEB), 'CDJ900NXSVirtual'))
|
|
args = ap.parse_args()
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods'))
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods', 'cdj900nxs'))
|
|
set_release(args.repo)
|
|
import patch_cdj900nxs as p
|
|
import hotcuekeys as hk
|
|
import cuecolours
|
|
with open(os.path.join(args.repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
if sha(stock) != p.APP_SHA256:
|
|
raise SystemExit('not the 1.31 application')
|
|
|
|
global ASSETS
|
|
ASSETS = p.assets
|
|
_, labels = hk.routine(hk.DECODER, cuecolours.HOT_TABLE)
|
|
w, h, on = hk.box_pixels(stock, hk.RES_QUANTIZE, 0xF800, 0x0000)
|
|
boxes = [{'addr': labels['PIX_ON'], 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xF800, 'ink': 0x0000,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': labels['PIX_OFF'], 'len': len(on), 'res': hk.RES_QUANTIZE_OFF, 'bg': 0x0000, 'ink': 0x4208,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': hk.PIX_LOAD, 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xFC60, 'ink': 0x0000,
|
|
'text': 'LOADING'}]
|
|
|
|
combos = [c for n in range(1, len(OURS) + 1) for c in itertools.combinations(OURS, n) if valid(c)]
|
|
variants, chunks = {}, {}
|
|
|
|
def encode(ops):
|
|
enc = []
|
|
for o in ops:
|
|
if o['op'] == 'data':
|
|
key = sha(o['data'])[:16]
|
|
chunks[key] = base64.b64encode(o['data']).decode()
|
|
enc.append(['data', o['addr'], key])
|
|
elif o['op'] == 'copy':
|
|
enc.append(['copy', o['addr'], o['src'], o['len']])
|
|
elif o['op'] == 'assets':
|
|
enc.append(['assets', o['addr'], o['len']])
|
|
else:
|
|
enc.append(['box', o['addr'], o['res'], o['bg'], o['ink'], o['text']])
|
|
return enc
|
|
|
|
# arnoldasr's mods, split up: per combination their bytes, but the asset
|
|
# blob (it starts with a copy of the firmware's resource table) made by
|
|
# the browser from the user's own application
|
|
upstream, up_apps = {}, {(): stock}
|
|
for k in range(1, len(p.UPSTREAM) + 1):
|
|
for up in itertools.combinations(p.UPSTREAM, k):
|
|
app = p.patch_upstream(stock, list(up))
|
|
# the asset blob itself (code of ours after it, advanced_waveforms'
|
|
# fix, goes as data)
|
|
blob_len = min(len(ASSETS(stock)[0]), len(app) - (p.DATA_BASE - BASE))
|
|
masked = bytearray(app)
|
|
masked[p.DATA_BASE - BASE:p.DATA_BASE - BASE + blob_len] = bytes(blob_len)
|
|
ops = ops_for(stock, bytes(masked), [])
|
|
ops.append({'op': 'assets', 'addr': p.DATA_BASE, 'len': blob_len})
|
|
if apply_ops(stock, stock, ops, hk, len(app)) != app:
|
|
raise SystemExit('%s: the operations do not give the patcher result' % ','.join(up))
|
|
upstream[','.join(up)] = {'size': len(app), 'ops': encode(ops)}
|
|
up_apps[up] = app
|
|
print('%-40s %3d operations' % (','.join(up), len(ops)))
|
|
# stage 1 only: the browser checks its result against these; stage 2 (ours)
|
|
# runs there as Python, the patcher's own code (builder/py/bundle.json)
|
|
expect = {','.join(up): sha(app) for up, app in up_apps.items() if up}
|
|
bundle = python_bundle(args.repo)
|
|
with open(os.path.join(WEB, 'builder', 'py', 'bundle.json'), 'w', encoding='utf-8', newline='\n') as f:
|
|
json.dump(bundle, f, separators=(',', ':'))
|
|
# stage 2 as the browser runs it: patch_ours on stage 1's application
|
|
for names in (['credit'], list(MODS_ALL), ['phasemeter'], ['advanced_waveforms', 'cuecolours', 'cuenames']):
|
|
up = tuple(n for n in p.UPSTREAM if n in names)
|
|
if p.patch_ours(up_apps[up], stock, p.check_names(names)) != p.patch(stock, names):
|
|
raise SystemExit('%s: stage 1 + stage 2 differs from the patcher' % ','.join(names))
|
|
print('checked: stage 1 data + stage 2 Python give the patcher bytes; %d Python files' % len(bundle))
|
|
|
|
glyphs = {c: rows for c, rows in hk.GLYPHS.items()}
|
|
data = {
|
|
'stockUpdSha256': '', # filled from the patcher's upd.py below
|
|
'stockAppSha256': p.APP_SHA256,
|
|
'appSize': p.APP_SIZE,
|
|
'dataBase': p.DATA_BASE,
|
|
'upstream': upstream,
|
|
'expect': expect,
|
|
'chunks': chunks,
|
|
}
|
|
sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher'))
|
|
import upd
|
|
data['stockUpdSha256'] = upd.STOCK_UPD_SHA256
|
|
for key, b64 in chunks.items(): # no firmware in what we ship
|
|
blob = base64.b64decode(b64)
|
|
for i in range(0, max(0, len(blob) - 47), 8):
|
|
win = blob[i:i + 48]
|
|
if len(set(win)) > 2 and stock.find(win) >= 0:
|
|
raise SystemExit('chunk %s holds firmware bytes (at %d)' % (key, i))
|
|
out = os.path.join(WEB, 'builder', 'data.js')
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write('// Generated by tools/make_builder_data.py -- do not edit.\n')
|
|
f.write('window.BUILDER_DATA = ')
|
|
json.dump(data, f, separators=(',', ':'))
|
|
f.write(';\n')
|
|
print('wrote %s (%d bytes, %d variants, %d chunks)' % (out, os.path.getsize(out), len(upstream), len(chunks)))
|
|
write_versions(p)
|
|
|
|
|
|
def write_versions(p):
|
|
"""assets/versions.js: the site's version and changelog (tools/changelog.json)
|
|
and every mod's version as patch_cdj900nxs.VERSIONS has it."""
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
log = json.load(f)
|
|
data = {'site': log['site'][0]['version'], 'date': log['site'][0]['date'], 'log': log['site'],
|
|
'mods': dict(p.VERSIONS), 'modlog': log.get('mods', {})}
|
|
out = os.path.join(WEB, 'assets', 'versions.js')
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write("// Generated by tools/make_builder_data.py from tools/changelog.json and the mods' versions.\n")
|
|
f.write('window.CDJ_VERSIONS = ' + json.dumps(data, indent=1, ensure_ascii=False) + ';\n')
|
|
print('wrote %s (site %s)' % (out, data['site']))
|
|
# the pages load the site's own CSS and scripts with ?v=<site version>, so
|
|
# a browser takes the new ones after an update
|
|
import re
|
|
for name in sorted(os.listdir(WEB)):
|
|
if not name.endswith('.html'):
|
|
continue
|
|
path = os.path.join(WEB, name)
|
|
with open(path, encoding='utf-8') as f:
|
|
html = f.read()
|
|
new = re.sub(r'(href|src)="((?:style\.css|script\.js|assets/versions\.js|builder/[a-z]+\.js))(\?v=[^"]*)?"',
|
|
lambda m: '%s="%s?v=%s"' % (m.group(1), m.group(2), data['site']), html)
|
|
new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], new) # the boot text
|
|
if new != html:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped %s with ?v=%s' % (name, data['site']))
|
|
ui = os.path.join(WEB, 'builder', 'ui.js')
|
|
with open(ui, encoding='utf-8') as f:
|
|
js = f.read()
|
|
new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], js)
|
|
if new != js:
|
|
with open(ui, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped builder/ui.js with Pionaarr Mods %s' % data['site'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|