- The flash bar is a plain card in the page flow (no sticky/floating). - Head, legend and "What each mod takes" tidied: values no longer break mid-number, sizes like "< 0.1 KB" stay on one line, the shared row is set apart; on phones each mod's name and size sit over a full-width bar. - The list stays open when a mod is ticked or unticked. - Phones: each mod's tags and flash share sit under its text instead of spilling out of the checkbox column. - make_builder_data.py: the boot text stamp leaves the release tags' "New in Pionaarr Mods 2.0.0" alone. - Boot text Pionaarr Mods 2.0.1. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
416 lines
19 KiB
Python
416 lines
19 KiB
Python
#!/usr/bin/env python3
|
|
# SPDX-License-Identifier: GPL-2.0-or-later
|
|
"""Writes builder/data.js, the data the web builder (builder.html) applies.
|
|
|
|
usage: make_builder_data.py [--repo ../CDJ900NXSVirtual]
|
|
|
|
Our mods (beatjump, cuecolours, hotcuekeys, ledsaver, keynotation, credit) are made by the emulator
|
|
checkout's mods/patch_cdj900nxs.py for every combination the builder offers;
|
|
their bytes do not depend on which of arnoldasr's mods are chosen (checked
|
|
below). For each combination the bytes our mods change in the application
|
|
go into data.js, except what is the player's own firmware:
|
|
|
|
copy the UTILITY item table (hotcuekeys, ledsaver and keynotation extend a
|
|
copy of it)
|
|
is copied from the user's own application in the browser;
|
|
box the HOT KEYS box is the QUANTIZE label's image recoloured with
|
|
HOT KEYS written on it; the browser makes it from the user's image.
|
|
|
|
arnoldasr's mods (credited on the site; advanced_waveforms now ours, based on his), as patch_cdj900nxs.py splits them
|
|
up, are in data.js the same way: per combination of extended_bpm, phasemeter
|
|
and advanced_waveforms the bytes they change, except the asset blob at 0x08FC2D00,
|
|
which starts with a copy of the firmware's resource table: the browser makes
|
|
it from the user's application (the port of patch_cdj900nxs.assets()).
|
|
The SHA-256 of the finished application for every combination is in data.js
|
|
too; the browser checks its result against it.
|
|
|
|
Before data.js is written, the browser's way of applying the data is redone
|
|
here in Python for every combination of arnoldasr's mods and ours and
|
|
compared byte for byte with patch_cdj900nxs.py's result.
|
|
"""
|
|
import argparse
|
|
import base64
|
|
import hashlib
|
|
import itertools
|
|
import json
|
|
import os
|
|
import struct
|
|
import sys
|
|
|
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
|
WEB = os.path.dirname(HERE)
|
|
OURS = ('beatjump', 'cuecolours', 'hotcuekeys', 'ledsaver', 'keynotation', 'credit')
|
|
NEEDS = {'hotcuekeys': 'cuecolours', 'beatjump': 'hotcuekeys'}
|
|
UT_TABLE_STOCK = 0x08C6C8DC
|
|
UT_TABLE_LEN = 17 * 52
|
|
BASE = 0x08000000
|
|
ASSETS = None
|
|
|
|
|
|
def sha(b):
|
|
return hashlib.sha256(b).hexdigest()
|
|
|
|
|
|
def valid(combo):
|
|
return all(NEEDS[m] in combo for m in combo if m in NEEDS)
|
|
|
|
|
|
def runs(base, final):
|
|
"""[(address, bytes)] where final differs from base (short equal gaps joined)."""
|
|
n = len(final)
|
|
base = base + bytes(n - len(base))
|
|
out, i = [], 0
|
|
while i < n:
|
|
if base[i] != final[i]:
|
|
j = i
|
|
while j < n and (base[j] != final[j] or (j + 16 < n and base[j:j + 16] != final[j:j + 16])):
|
|
j += 1
|
|
out.append((BASE + i, final[i:j]))
|
|
i = j
|
|
else:
|
|
i += 1
|
|
return out
|
|
|
|
|
|
def spans(idx):
|
|
"""[(lo, hi)] of the runs of consecutive numbers in idx (sorted)."""
|
|
out = []
|
|
for i in idx:
|
|
if out and out[-1][1] == i:
|
|
out[-1] = (out[-1][0], i + 1)
|
|
else:
|
|
out.append((i, i + 1))
|
|
return out
|
|
|
|
|
|
def ops_for(stock, final, boxes):
|
|
"""The browser's operations for one combination: data, copy, box."""
|
|
table = stock[UT_TABLE_STOCK - BASE:UT_TABLE_STOCK - BASE + UT_TABLE_LEN]
|
|
out = [dict(box, op='box') for box in boxes] # whole, outside the runs
|
|
masked = bytearray(final)
|
|
padded = stock + bytes(len(final) - len(stock))
|
|
for box in boxes:
|
|
off = box['addr'] - BASE
|
|
masked[off:off + box['len']] = padded[off:off + box['len']]
|
|
for addr, data in runs(stock, bytes(masked)):
|
|
# (start, end, ops) inside this run: a copy of the table, a few of its
|
|
# items changed (beatjump renames SCREEN SAVER), is the copy and then
|
|
# our bytes over it
|
|
cuts = []
|
|
k = data.find(table[:52])
|
|
while k >= 0:
|
|
seg = data[k:k + UT_TABLE_LEN]
|
|
diff = [i for i in range(len(seg)) if seg[i] != table[i]] if len(seg) == UT_TABLE_LEN else None
|
|
if diff is not None and len(diff) <= 2 * 52:
|
|
ops = [{'op': 'copy', 'addr': addr + k, 'src': UT_TABLE_STOCK, 'len': UT_TABLE_LEN}]
|
|
for lo, hi in spans(diff):
|
|
ops.append({'op': 'data', 'addr': addr + k + lo, 'data': seg[lo:hi]})
|
|
cuts.append((k, k + UT_TABLE_LEN, ops))
|
|
k = data.find(table[:52], k + UT_TABLE_LEN)
|
|
else:
|
|
k = data.find(table[:52], k + 1)
|
|
cuts.sort(key=lambda c: c[0])
|
|
pos = 0
|
|
for lo, hi, ops in cuts:
|
|
if lo > pos:
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:lo]})
|
|
out.extend(ops)
|
|
pos = hi
|
|
if pos < len(data):
|
|
out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:]})
|
|
return out
|
|
|
|
|
|
def apply_ops(app, stock, ops, hk, size):
|
|
"""What builder.js does with the operations (redone here to check them):
|
|
the application grown with zeros to `size` first."""
|
|
out = bytearray(app) + bytes(max(0, size - len(app)))
|
|
for o in ops:
|
|
off = o['addr'] - BASE
|
|
if o['op'] == 'data':
|
|
out[off:off + len(o['data'])] = o['data']
|
|
elif o['op'] == 'copy':
|
|
out[off:off + o['len']] = stock[o['src'] - BASE:o['src'] - BASE + o['len']]
|
|
elif o['op'] == 'assets':
|
|
out[off:off + o['len']] = ASSETS(stock)[0][:o['len']]
|
|
else:
|
|
_, _, px = hk.box_pixels(stock, o['res'], o['bg'], o['ink'], o['text'])
|
|
out[off:off + o['len']] = px
|
|
return bytes(out)
|
|
|
|
|
|
MODS_ALL = ('extended_bpm', 'phasemeter', 'advanced_waveforms', 'beatjump', 'cuecolours', 'hotcuekeys',
|
|
'ledsaver', 'keynotation', 'usbautoload', 'quicksearch', 'cuenames', 'played', 'endwarn', 'wide_waveform',
|
|
'browseinfo', 'divhold', 'flacplay', 'exfatfs', 'credit')
|
|
FLASH_NEEDS = {'beatjump': ('cuecolours', 'hotcuekeys'), 'hotcuekeys': ('cuecolours',), 'cuenames': ('cuecolours',)}
|
|
FLASH_CACHE = os.path.join(HERE, 'flash_sizes.json')
|
|
|
|
|
|
def _packed_len(job):
|
|
"""(repo, names) -> the LZSS stream's length of the application with those mods (one process each)."""
|
|
repo, names = job
|
|
sys.path[:0] = [os.path.join(repo, 'mods'), os.path.join(repo, 'mods', 'cdj900nxs'),
|
|
os.path.join(repo, 'scripts', 'firmware')]
|
|
import patch_cdj900nxs as p
|
|
import lzss_encode
|
|
with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
app = p.patch(stock, list(names)) if names else stock
|
|
return names, sha(app), len(lzss_encode.pack(app))
|
|
|
|
|
|
def flash_sizes(repo, p):
|
|
"""What each mod takes of the player's flash: the application is stored LZSS-packed in
|
|
0x40000-0x7E0000 (upd.APP_END - APP_START bytes). A mod's share is the packed size with it
|
|
(and what it needs) minus without it; the sum over a choice is an estimate (the packer finds
|
|
a little more or less to share), the exact number the build tells. Packing takes a minute a
|
|
run, so results are cached by the application's SHA-256 (tools/flash_sizes.json)."""
|
|
sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher'))
|
|
import upd
|
|
from multiprocessing import Pool
|
|
try:
|
|
with open(FLASH_CACHE, encoding='utf-8') as f:
|
|
cache = json.load(f)
|
|
except (OSError, ValueError):
|
|
cache = {}
|
|
with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
def removed(m): # m and every mod that needs it, directly or not
|
|
out, grow = {m}, True
|
|
while grow:
|
|
grow = False
|
|
for n, d in FLASH_NEEDS.items():
|
|
if n not in out and out & set(d):
|
|
out.add(n)
|
|
grow = True
|
|
return out
|
|
jobs = {(): None, MODS_ALL: None}
|
|
for m in MODS_ALL:
|
|
jobs[tuple(n for n in MODS_ALL if n not in removed(m))] = None
|
|
d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ()))
|
|
jobs[d] = None
|
|
jobs[tuple(n for n in MODS_ALL if n in d + (m,))] = None
|
|
todo, known = [], {}
|
|
for names in jobs:
|
|
app_sha = sha(p.patch(stock, list(names)) if names else stock)
|
|
if app_sha in cache:
|
|
known[names] = cache[app_sha]
|
|
else:
|
|
todo.append((repo, names))
|
|
if todo:
|
|
print('packing %d applications for the flash sizes (about a minute each, in parallel)' % len(todo))
|
|
with Pool(min(len(todo), max(1, (os.cpu_count() or 2) - 1))) as pool:
|
|
for names, app_sha, n in pool.imap_unordered(_packed_len, todo):
|
|
cache[app_sha] = n
|
|
known[names] = n
|
|
print(' %-60s %d' % (','.join(names) or 'stock', n))
|
|
with open(FLASH_CACHE, 'w', encoding='utf-8', newline=chr(10)) as f:
|
|
json.dump(cache, f, indent=1, sort_keys=True)
|
|
region = lambda n: len(upd.pack_region(bytes(n)))
|
|
# The packer shares a lot between the mods (alone each costs 7-30 KB, all together 50 KB), so a
|
|
# sum of single sizes is far off. What every mod brings anyway (the frame: the smallest mod alone,
|
|
# the boot text, costs it) is counted once; the rest is split by each mod's marginal size in the
|
|
# full build (all mods packed minus all but it, and what needs it). Exact with none and with all.
|
|
alone = {}
|
|
for m in MODS_ALL:
|
|
d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ()))
|
|
alone[m] = known[tuple(n for n in MODS_ALL if n in d + (m,))] - known[d]
|
|
frame = min(v for m, v in alone.items() if m not in FLASH_NEEDS) # (one that needs others adds to them)
|
|
marg = {}
|
|
for m in sorted(MODS_ALL, key=lambda m: len(removed(m))):
|
|
rest = tuple(n for n in MODS_ALL if n not in removed(m))
|
|
marg[m] = max(0, known[MODS_ALL] - known[rest] - sum(marg[n] for n in removed(m) - {m}))
|
|
total = region(known[MODS_ALL]) - region(known[()]) - frame
|
|
k = total / float(sum(marg.values()) or 1)
|
|
mods = {m: max(1, int(round(marg[m] * k))) for m in MODS_ALL}
|
|
out = {'size': upd.APP_END - upd.APP_START, 'stock': region(known[()]),
|
|
'all': region(known[MODS_ALL]), 'common': frame, 'mods': mods}
|
|
print('flash: %d of %d with no mods, %d with all; shares %d + frame %d'
|
|
% (out['stock'], out['size'], out['all'], sum(mods.values()), out['common']))
|
|
print(' ' + ', '.join('%s %d' % kv for kv in sorted(mods.items(), key=lambda kv: -kv[1])))
|
|
return out
|
|
|
|
|
|
def python_bundle(repo):
|
|
"""Our patch code for the browser: mods/patch_cdj900nxs.py and every module
|
|
in mods/cdj900nxs (not arnoldasr's checkout, not the tests)."""
|
|
files = {'mods/patch_cdj900nxs.py': os.path.join(repo, 'mods', 'patch_cdj900nxs.py')}
|
|
d = os.path.join(repo, 'mods', 'cdj900nxs')
|
|
for n in sorted(os.listdir(d)):
|
|
if n.endswith('.py'):
|
|
files['mods/cdj900nxs/' + n] = os.path.join(d, n)
|
|
out = {}
|
|
for rel, path in files.items():
|
|
with open(path, encoding='utf-8') as f:
|
|
out[rel] = f.read()
|
|
return out
|
|
|
|
|
|
def set_release(repo):
|
|
"""The boot screen (the credit mod) names the Pionaarr Mods release: always
|
|
the site's version (tools/changelog.json), written into credit.py's
|
|
RELEASE before anything is built."""
|
|
import re
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
site = json.load(f)['site'][0]['version']
|
|
path = os.path.join(repo, 'mods', 'cdj900nxs', 'credit.py')
|
|
with open(path, encoding='utf-8') as f:
|
|
src = f.read()
|
|
new = re.sub(r"^RELEASE = '[^']*'", "RELEASE = '%s'" % site, src, count=1, flags=re.M)
|
|
if new != src:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('set credit.RELEASE to %s (the boot screen)' % site)
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument('--repo', default=os.path.join(os.path.dirname(WEB), 'CDJ900NXSVirtual'))
|
|
args = ap.parse_args()
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods'))
|
|
sys.path.insert(0, os.path.join(args.repo, 'mods', 'cdj900nxs'))
|
|
set_release(args.repo)
|
|
import patch_cdj900nxs as p
|
|
import hotcuekeys as hk
|
|
import cuecolours
|
|
with open(os.path.join(args.repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f:
|
|
stock = f.read()[:p.APP_SIZE]
|
|
if sha(stock) != p.APP_SHA256:
|
|
raise SystemExit('not the 1.31 application')
|
|
|
|
global ASSETS
|
|
ASSETS = p.assets
|
|
_, labels = hk.routine(hk.DECODER, cuecolours.HOT_TABLE)
|
|
w, h, on = hk.box_pixels(stock, hk.RES_QUANTIZE, 0xF800, 0x0000)
|
|
boxes = [{'addr': labels['PIX_ON'], 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xF800, 'ink': 0x0000,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': labels['PIX_OFF'], 'len': len(on), 'res': hk.RES_QUANTIZE_OFF, 'bg': 0x0000, 'ink': 0x4208,
|
|
'text': 'HOT KEYS'},
|
|
{'addr': hk.PIX_LOAD, 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xFC60, 'ink': 0x0000,
|
|
'text': 'LOADING'}]
|
|
|
|
combos = [c for n in range(1, len(OURS) + 1) for c in itertools.combinations(OURS, n) if valid(c)]
|
|
variants, chunks = {}, {}
|
|
|
|
def encode(ops):
|
|
enc = []
|
|
for o in ops:
|
|
if o['op'] == 'data':
|
|
key = sha(o['data'])[:16]
|
|
chunks[key] = base64.b64encode(o['data']).decode()
|
|
enc.append(['data', o['addr'], key])
|
|
elif o['op'] == 'copy':
|
|
enc.append(['copy', o['addr'], o['src'], o['len']])
|
|
elif o['op'] == 'assets':
|
|
enc.append(['assets', o['addr'], o['len']])
|
|
else:
|
|
enc.append(['box', o['addr'], o['res'], o['bg'], o['ink'], o['text']])
|
|
return enc
|
|
|
|
# arnoldasr's mods, split up: per combination their bytes, but the asset
|
|
# blob (it starts with a copy of the firmware's resource table) made by
|
|
# the browser from the user's own application
|
|
upstream, up_apps = {}, {(): stock}
|
|
for k in range(1, len(p.UPSTREAM) + 1):
|
|
for up in itertools.combinations(p.UPSTREAM, k):
|
|
app = p.patch_upstream(stock, list(up))
|
|
# the asset blob itself (code of ours after it, advanced_waveforms'
|
|
# fix, goes as data)
|
|
blob_len = min(len(ASSETS(stock)[0]), len(app) - (p.DATA_BASE - BASE))
|
|
masked = bytearray(app)
|
|
masked[p.DATA_BASE - BASE:p.DATA_BASE - BASE + blob_len] = bytes(blob_len)
|
|
ops = ops_for(stock, bytes(masked), [])
|
|
ops.append({'op': 'assets', 'addr': p.DATA_BASE, 'len': blob_len})
|
|
if apply_ops(stock, stock, ops, hk, len(app)) != app:
|
|
raise SystemExit('%s: the operations do not give the patcher result' % ','.join(up))
|
|
upstream[','.join(up)] = {'size': len(app), 'ops': encode(ops)}
|
|
up_apps[up] = app
|
|
print('%-40s %3d operations' % (','.join(up), len(ops)))
|
|
# stage 1 only: the browser checks its result against these; stage 2 (ours)
|
|
# runs there as Python, the patcher's own code (builder/py/bundle.json)
|
|
expect = {','.join(up): sha(app) for up, app in up_apps.items() if up}
|
|
bundle = python_bundle(args.repo)
|
|
with open(os.path.join(WEB, 'builder', 'py', 'bundle.json'), 'w', encoding='utf-8', newline='\n') as f:
|
|
json.dump(bundle, f, separators=(',', ':'))
|
|
# stage 2 as the browser runs it: patch_ours on stage 1's application
|
|
for names in (['credit'], list(MODS_ALL), ['phasemeter'], ['advanced_waveforms', 'cuecolours', 'cuenames']):
|
|
up = tuple(n for n in p.UPSTREAM if n in names)
|
|
if p.patch_ours(up_apps[up], stock, p.check_names(names)) != p.patch(stock, names):
|
|
raise SystemExit('%s: stage 1 + stage 2 differs from the patcher' % ','.join(names))
|
|
print('checked: stage 1 data + stage 2 Python give the patcher bytes; %d Python files' % len(bundle))
|
|
|
|
glyphs = {c: rows for c, rows in hk.GLYPHS.items()}
|
|
data = {
|
|
'stockUpdSha256': '', # filled from the patcher's upd.py below
|
|
'stockAppSha256': p.APP_SHA256,
|
|
'appSize': p.APP_SIZE,
|
|
'dataBase': p.DATA_BASE,
|
|
'upstream': upstream,
|
|
'expect': expect,
|
|
'chunks': chunks,
|
|
}
|
|
sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher'))
|
|
import upd
|
|
data['stockUpdSha256'] = upd.STOCK_UPD_SHA256
|
|
data['flash'] = flash_sizes(args.repo, p)
|
|
for key, b64 in chunks.items(): # no firmware in what we ship
|
|
blob = base64.b64decode(b64)
|
|
for i in range(0, max(0, len(blob) - 47), 8):
|
|
win = blob[i:i + 48]
|
|
if len(set(win)) > 2 and stock.find(win) >= 0:
|
|
raise SystemExit('chunk %s holds firmware bytes (at %d)' % (key, i))
|
|
out = os.path.join(WEB, 'builder', 'data.js')
|
|
os.makedirs(os.path.dirname(out), exist_ok=True)
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write('// Generated by tools/make_builder_data.py -- do not edit.\n')
|
|
f.write('window.BUILDER_DATA = ')
|
|
json.dump(data, f, separators=(',', ':'))
|
|
f.write(';\n')
|
|
print('wrote %s (%d bytes, %d variants, %d chunks)' % (out, os.path.getsize(out), len(upstream), len(chunks)))
|
|
write_versions(p)
|
|
|
|
|
|
def write_versions(p):
|
|
"""assets/versions.js: the site's version and changelog (tools/changelog.json)
|
|
and every mod's version as patch_cdj900nxs.VERSIONS has it."""
|
|
with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f:
|
|
log = json.load(f)
|
|
data = {'site': log['site'][0]['version'], 'date': log['site'][0]['date'], 'log': log['site'],
|
|
'mods': dict(p.VERSIONS), 'modlog': log.get('mods', {})}
|
|
out = os.path.join(WEB, 'assets', 'versions.js')
|
|
with open(out, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write("// Generated by tools/make_builder_data.py from tools/changelog.json and the mods' versions.\n")
|
|
f.write('window.CDJ_VERSIONS = ' + json.dumps(data, indent=1, ensure_ascii=False) + ';\n')
|
|
print('wrote %s (site %s)' % (out, data['site']))
|
|
# the pages load the site's own CSS and scripts with ?v=<site version>, so
|
|
# a browser takes the new ones after an update
|
|
import re
|
|
# the boot text follows the site's version; a release tag's "New in
|
|
# Pionaarr Mods 2.0.0" names the release the mod came with and stays
|
|
BOOT_TEXT = r'(?<!New in )Pionaarr Mods \d+\.\d+(\.\d+)?'
|
|
for name in sorted(os.listdir(WEB)):
|
|
if not name.endswith('.html'):
|
|
continue
|
|
path = os.path.join(WEB, name)
|
|
with open(path, encoding='utf-8') as f:
|
|
html = f.read()
|
|
new = re.sub(r'(href|src)="((?:style\.css|script\.js|assets/versions\.js|builder/[a-z]+\.js))(\?v=[^"]*)?"',
|
|
lambda m: '%s="%s?v=%s"' % (m.group(1), m.group(2), data['site']), html)
|
|
new = re.sub(BOOT_TEXT, 'Pionaarr Mods ' + data['site'], new) # the boot text
|
|
if new != html:
|
|
with open(path, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped %s with ?v=%s' % (name, data['site']))
|
|
ui = os.path.join(WEB, 'builder', 'ui.js')
|
|
with open(ui, encoding='utf-8') as f:
|
|
js = f.read()
|
|
new = re.sub(BOOT_TEXT, 'Pionaarr Mods ' + data['site'], js)
|
|
if new != js:
|
|
with open(ui, 'w', encoding='utf-8', newline='\n') as f:
|
|
f.write(new)
|
|
print('stamped builder/ui.js with Pionaarr Mods %s' % data['site'])
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|