#!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0-or-later """Writes builder/data.js, the data the web builder (builder.html) applies. usage: make_builder_data.py [--repo ../CDJ900NXSVirtual] Our mods (beatjump, cuecolours, hotcuekeys, ledsaver, keynotation, credit) are made by the emulator checkout's mods/patch_cdj900nxs.py for every combination the builder offers; their bytes do not depend on which of arnoldasr's mods are chosen (checked below). For each combination the bytes our mods change in the application go into data.js, except what is the player's own firmware: copy the UTILITY item table (hotcuekeys, ledsaver and keynotation extend a copy of it) is copied from the user's own application in the browser; box the HOT KEYS box is the QUANTIZE label's image recoloured with HOT KEYS written on it; the browser makes it from the user's image. arnoldasr's mods (credited on the site; advanced_waveforms now ours, based on his), as patch_cdj900nxs.py splits them up, are in data.js the same way: per combination of extended_bpm, phasemeter and advanced_waveforms the bytes they change, except the asset blob at 0x08FC2D00, which starts with a copy of the firmware's resource table: the browser makes it from the user's application (the port of patch_cdj900nxs.assets()). The SHA-256 of the finished application for every combination is in data.js too; the browser checks its result against it. Before data.js is written, the browser's way of applying the data is redone here in Python for every combination of arnoldasr's mods and ours and compared byte for byte with patch_cdj900nxs.py's result. """ import argparse import base64 import hashlib import itertools import json import os import struct import sys HERE = os.path.dirname(os.path.abspath(__file__)) WEB = os.path.dirname(HERE) OURS = ('beatjump', 'cuecolours', 'hotcuekeys', 'ledsaver', 'keynotation', 'credit') NEEDS = {'hotcuekeys': 'cuecolours', 'beatjump': 'hotcuekeys'} UT_TABLE_STOCK = 0x08C6C8DC UT_TABLE_LEN = 17 * 52 BASE = 0x08000000 ASSETS = None def sha(b): return hashlib.sha256(b).hexdigest() def valid(combo): return all(NEEDS[m] in combo for m in combo if m in NEEDS) def runs(base, final): """[(address, bytes)] where final differs from base (short equal gaps joined).""" n = len(final) base = base + bytes(n - len(base)) out, i = [], 0 while i < n: if base[i] != final[i]: j = i while j < n and (base[j] != final[j] or (j + 16 < n and base[j:j + 16] != final[j:j + 16])): j += 1 out.append((BASE + i, final[i:j])) i = j else: i += 1 return out def spans(idx): """[(lo, hi)] of the runs of consecutive numbers in idx (sorted).""" out = [] for i in idx: if out and out[-1][1] == i: out[-1] = (out[-1][0], i + 1) else: out.append((i, i + 1)) return out def ops_for(stock, final, boxes): """The browser's operations for one combination: data, copy, box.""" table = stock[UT_TABLE_STOCK - BASE:UT_TABLE_STOCK - BASE + UT_TABLE_LEN] out = [dict(box, op='box') for box in boxes] # whole, outside the runs masked = bytearray(final) padded = stock + bytes(len(final) - len(stock)) for box in boxes: off = box['addr'] - BASE masked[off:off + box['len']] = padded[off:off + box['len']] for addr, data in runs(stock, bytes(masked)): # (start, end, ops) inside this run: a copy of the table, a few of its # items changed (beatjump renames SCREEN SAVER), is the copy and then # our bytes over it cuts = [] k = data.find(table[:52]) while k >= 0: seg = data[k:k + UT_TABLE_LEN] diff = [i for i in range(len(seg)) if seg[i] != table[i]] if len(seg) == UT_TABLE_LEN else None if diff is not None and len(diff) <= 2 * 52: ops = [{'op': 'copy', 'addr': addr + k, 'src': UT_TABLE_STOCK, 'len': UT_TABLE_LEN}] for lo, hi in spans(diff): ops.append({'op': 'data', 'addr': addr + k + lo, 'data': seg[lo:hi]}) cuts.append((k, k + UT_TABLE_LEN, ops)) k = data.find(table[:52], k + UT_TABLE_LEN) else: k = data.find(table[:52], k + 1) cuts.sort(key=lambda c: c[0]) pos = 0 for lo, hi, ops in cuts: if lo > pos: out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:lo]}) out.extend(ops) pos = hi if pos < len(data): out.append({'op': 'data', 'addr': addr + pos, 'data': data[pos:]}) return out def apply_ops(app, stock, ops, hk, size): """What builder.js does with the operations (redone here to check them): the application grown with zeros to `size` first.""" out = bytearray(app) + bytes(max(0, size - len(app))) for o in ops: off = o['addr'] - BASE if o['op'] == 'data': out[off:off + len(o['data'])] = o['data'] elif o['op'] == 'copy': out[off:off + o['len']] = stock[o['src'] - BASE:o['src'] - BASE + o['len']] elif o['op'] == 'assets': out[off:off + o['len']] = ASSETS(stock)[0][:o['len']] else: _, _, px = hk.box_pixels(stock, o['res'], o['bg'], o['ink'], o['text']) out[off:off + o['len']] = px return bytes(out) MODS_ALL = ('extended_bpm', 'phasemeter', 'advanced_waveforms', 'beatjump', 'cuecolours', 'hotcuekeys', 'ledsaver', 'keynotation', 'usbautoload', 'quicksearch', 'cuenames', 'played', 'endwarn', 'wide_waveform', 'browseinfo', 'divhold', 'credit') FLASH_NEEDS = {'beatjump': ('cuecolours', 'hotcuekeys'), 'hotcuekeys': ('cuecolours',), 'cuenames': ('cuecolours',)} FLASH_CACHE = os.path.join(HERE, 'flash_sizes.json') def _packed_len(job): """(repo, names) -> the LZSS stream's length of the application with those mods (one process each).""" repo, names = job sys.path[:0] = [os.path.join(repo, 'mods'), os.path.join(repo, 'mods', 'cdj900nxs'), os.path.join(repo, 'scripts', 'firmware')] import patch_cdj900nxs as p import lzss_encode with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f: stock = f.read()[:p.APP_SIZE] app = p.patch(stock, list(names)) if names else stock return names, sha(app), len(lzss_encode.pack(app)) def flash_sizes(repo, p): """What each mod takes of the player's flash: the application is stored LZSS-packed in 0x40000-0x7E0000 (upd.APP_END - APP_START bytes). A mod's share is the packed size with it (and what it needs) minus without it; the sum over a choice is an estimate (the packer finds a little more or less to share), the exact number the build tells. Packing takes a minute a run, so results are cached by the application's SHA-256 (tools/flash_sizes.json).""" sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher')) import upd from multiprocessing import Pool try: with open(FLASH_CACHE, encoding='utf-8') as f: cache = json.load(f) except (OSError, ValueError): cache = {} with open(os.path.join(repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f: stock = f.read()[:p.APP_SIZE] def removed(m): # m and every mod that needs it, directly or not out, grow = {m}, True while grow: grow = False for n, d in FLASH_NEEDS.items(): if n not in out and out & set(d): out.add(n) grow = True return out jobs = {(): None, MODS_ALL: None} for m in MODS_ALL: jobs[tuple(n for n in MODS_ALL if n not in removed(m))] = None d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ())) jobs[d] = None jobs[tuple(n for n in MODS_ALL if n in d + (m,))] = None todo, known = [], {} for names in jobs: app_sha = sha(p.patch(stock, list(names)) if names else stock) if app_sha in cache: known[names] = cache[app_sha] else: todo.append((repo, names)) if todo: print('packing %d applications for the flash sizes (about a minute each, in parallel)' % len(todo)) with Pool(min(len(todo), max(1, (os.cpu_count() or 2) - 1))) as pool: for names, app_sha, n in pool.imap_unordered(_packed_len, todo): cache[app_sha] = n known[names] = n print(' %-60s %d' % (','.join(names) or 'stock', n)) with open(FLASH_CACHE, 'w', encoding='utf-8', newline=chr(10)) as f: json.dump(cache, f, indent=1, sort_keys=True) region = lambda n: len(upd.pack_region(bytes(n))) # The packer shares a lot between the mods (alone each costs 7-30 KB, all together 50 KB), so a # sum of single sizes is far off. What every mod brings anyway (the frame: the smallest mod alone, # the boot text, costs it) is counted once; the rest is split by each mod's marginal size in the # full build (all mods packed minus all but it, and what needs it). Exact with none and with all. alone = {} for m in MODS_ALL: d = tuple(n for n in MODS_ALL if n in FLASH_NEEDS.get(m, ())) alone[m] = known[tuple(n for n in MODS_ALL if n in d + (m,))] - known[d] frame = min(v for m, v in alone.items() if m not in FLASH_NEEDS) # (one that needs others adds to them) marg = {} for m in sorted(MODS_ALL, key=lambda m: len(removed(m))): rest = tuple(n for n in MODS_ALL if n not in removed(m)) marg[m] = max(0, known[MODS_ALL] - known[rest] - sum(marg[n] for n in removed(m) - {m})) total = region(known[MODS_ALL]) - region(known[()]) - frame k = total / float(sum(marg.values()) or 1) mods = {m: max(1, int(round(marg[m] * k))) for m in MODS_ALL} out = {'size': upd.APP_END - upd.APP_START, 'stock': region(known[()]), 'all': region(known[MODS_ALL]), 'common': frame, 'mods': mods} print('flash: %d of %d with no mods, %d with all; shares %d + frame %d' % (out['stock'], out['size'], out['all'], sum(mods.values()), out['common'])) print(' ' + ', '.join('%s %d' % kv for kv in sorted(mods.items(), key=lambda kv: -kv[1]))) return out def python_bundle(repo): """Our patch code for the browser: mods/patch_cdj900nxs.py and every module in mods/cdj900nxs (not arnoldasr's checkout, not the tests).""" files = {'mods/patch_cdj900nxs.py': os.path.join(repo, 'mods', 'patch_cdj900nxs.py')} d = os.path.join(repo, 'mods', 'cdj900nxs') for n in sorted(os.listdir(d)): if n.endswith('.py'): files['mods/cdj900nxs/' + n] = os.path.join(d, n) out = {} for rel, path in files.items(): with open(path, encoding='utf-8') as f: out[rel] = f.read() return out def set_release(repo): """The boot screen (the credit mod) names the Pionaarr Mods release: always the site's version (tools/changelog.json), written into credit.py's RELEASE before anything is built.""" import re with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f: site = json.load(f)['site'][0]['version'] path = os.path.join(repo, 'mods', 'cdj900nxs', 'credit.py') with open(path, encoding='utf-8') as f: src = f.read() new = re.sub(r"^RELEASE = '[^']*'", "RELEASE = '%s'" % site, src, count=1, flags=re.M) if new != src: with open(path, 'w', encoding='utf-8', newline='\n') as f: f.write(new) print('set credit.RELEASE to %s (the boot screen)' % site) def main(): ap = argparse.ArgumentParser() ap.add_argument('--repo', default=os.path.join(os.path.dirname(WEB), 'CDJ900NXSVirtual')) args = ap.parse_args() sys.path.insert(0, os.path.join(args.repo, 'mods')) sys.path.insert(0, os.path.join(args.repo, 'mods', 'cdj900nxs')) set_release(args.repo) import patch_cdj900nxs as p import hotcuekeys as hk import cuecolours with open(os.path.join(args.repo, 'extract', 'cdj900nxs', 'main_unpacked.bin'), 'rb') as f: stock = f.read()[:p.APP_SIZE] if sha(stock) != p.APP_SHA256: raise SystemExit('not the 1.31 application') global ASSETS ASSETS = p.assets _, labels = hk.routine(hk.DECODER, cuecolours.HOT_TABLE) w, h, on = hk.box_pixels(stock, hk.RES_QUANTIZE, 0xF800, 0x0000) boxes = [{'addr': labels['PIX_ON'], 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xF800, 'ink': 0x0000, 'text': 'HOT KEYS'}, {'addr': labels['PIX_OFF'], 'len': len(on), 'res': hk.RES_QUANTIZE_OFF, 'bg': 0x0000, 'ink': 0x4208, 'text': 'HOT KEYS'}, {'addr': hk.PIX_LOAD, 'len': len(on), 'res': hk.RES_QUANTIZE, 'bg': 0xFC60, 'ink': 0x0000, 'text': 'LOADING'}] combos = [c for n in range(1, len(OURS) + 1) for c in itertools.combinations(OURS, n) if valid(c)] variants, chunks = {}, {} def encode(ops): enc = [] for o in ops: if o['op'] == 'data': key = sha(o['data'])[:16] chunks[key] = base64.b64encode(o['data']).decode() enc.append(['data', o['addr'], key]) elif o['op'] == 'copy': enc.append(['copy', o['addr'], o['src'], o['len']]) elif o['op'] == 'assets': enc.append(['assets', o['addr'], o['len']]) else: enc.append(['box', o['addr'], o['res'], o['bg'], o['ink'], o['text']]) return enc # arnoldasr's mods, split up: per combination their bytes, but the asset # blob (it starts with a copy of the firmware's resource table) made by # the browser from the user's own application upstream, up_apps = {}, {(): stock} for k in range(1, len(p.UPSTREAM) + 1): for up in itertools.combinations(p.UPSTREAM, k): app = p.patch_upstream(stock, list(up)) # the asset blob itself (code of ours after it, advanced_waveforms' # fix, goes as data) blob_len = min(len(ASSETS(stock)[0]), len(app) - (p.DATA_BASE - BASE)) masked = bytearray(app) masked[p.DATA_BASE - BASE:p.DATA_BASE - BASE + blob_len] = bytes(blob_len) ops = ops_for(stock, bytes(masked), []) ops.append({'op': 'assets', 'addr': p.DATA_BASE, 'len': blob_len}) if apply_ops(stock, stock, ops, hk, len(app)) != app: raise SystemExit('%s: the operations do not give the patcher result' % ','.join(up)) upstream[','.join(up)] = {'size': len(app), 'ops': encode(ops)} up_apps[up] = app print('%-40s %3d operations' % (','.join(up), len(ops))) # stage 1 only: the browser checks its result against these; stage 2 (ours) # runs there as Python, the patcher's own code (builder/py/bundle.json) expect = {','.join(up): sha(app) for up, app in up_apps.items() if up} bundle = python_bundle(args.repo) with open(os.path.join(WEB, 'builder', 'py', 'bundle.json'), 'w', encoding='utf-8', newline='\n') as f: json.dump(bundle, f, separators=(',', ':')) # stage 2 as the browser runs it: patch_ours on stage 1's application for names in (['credit'], list(MODS_ALL), ['phasemeter'], ['advanced_waveforms', 'cuecolours', 'cuenames']): up = tuple(n for n in p.UPSTREAM if n in names) if p.patch_ours(up_apps[up], stock, p.check_names(names)) != p.patch(stock, names): raise SystemExit('%s: stage 1 + stage 2 differs from the patcher' % ','.join(names)) print('checked: stage 1 data + stage 2 Python give the patcher bytes; %d Python files' % len(bundle)) glyphs = {c: rows for c, rows in hk.GLYPHS.items()} data = { 'stockUpdSha256': '', # filled from the patcher's upd.py below 'stockAppSha256': p.APP_SHA256, 'appSize': p.APP_SIZE, 'dataBase': p.DATA_BASE, 'upstream': upstream, 'expect': expect, 'chunks': chunks, } sys.path.insert(0, os.path.join(os.path.dirname(WEB), 'CDJ900NXSPatcher')) import upd data['stockUpdSha256'] = upd.STOCK_UPD_SHA256 data['flash'] = flash_sizes(args.repo, p) for key, b64 in chunks.items(): # no firmware in what we ship blob = base64.b64decode(b64) for i in range(0, max(0, len(blob) - 47), 8): win = blob[i:i + 48] if len(set(win)) > 2 and stock.find(win) >= 0: raise SystemExit('chunk %s holds firmware bytes (at %d)' % (key, i)) out = os.path.join(WEB, 'builder', 'data.js') os.makedirs(os.path.dirname(out), exist_ok=True) with open(out, 'w', encoding='utf-8', newline='\n') as f: f.write('// Generated by tools/make_builder_data.py -- do not edit.\n') f.write('window.BUILDER_DATA = ') json.dump(data, f, separators=(',', ':')) f.write(';\n') print('wrote %s (%d bytes, %d variants, %d chunks)' % (out, os.path.getsize(out), len(upstream), len(chunks))) write_versions(p) def write_versions(p): """assets/versions.js: the site's version and changelog (tools/changelog.json) and every mod's version as patch_cdj900nxs.VERSIONS has it.""" with open(os.path.join(HERE, 'changelog.json'), encoding='utf-8') as f: log = json.load(f) data = {'site': log['site'][0]['version'], 'date': log['site'][0]['date'], 'log': log['site'], 'mods': dict(p.VERSIONS), 'modlog': log.get('mods', {})} out = os.path.join(WEB, 'assets', 'versions.js') with open(out, 'w', encoding='utf-8', newline='\n') as f: f.write("// Generated by tools/make_builder_data.py from tools/changelog.json and the mods' versions.\n") f.write('window.CDJ_VERSIONS = ' + json.dumps(data, indent=1, ensure_ascii=False) + ';\n') print('wrote %s (site %s)' % (out, data['site'])) # the pages load the site's own CSS and scripts with ?v=, so # a browser takes the new ones after an update import re for name in sorted(os.listdir(WEB)): if not name.endswith('.html'): continue path = os.path.join(WEB, name) with open(path, encoding='utf-8') as f: html = f.read() new = re.sub(r'(href|src)="((?:style\.css|script\.js|assets/versions\.js|builder/[a-z]+\.js))(\?v=[^"]*)?"', lambda m: '%s="%s?v=%s"' % (m.group(1), m.group(2), data['site']), html) new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], new) # the boot text if new != html: with open(path, 'w', encoding='utf-8', newline='\n') as f: f.write(new) print('stamped %s with ?v=%s' % (name, data['site'])) ui = os.path.join(WEB, 'builder', 'ui.js') with open(ui, encoding='utf-8') as f: js = f.read() new = re.sub(r'Pionaarr Mods \d+\.\d+(\.\d+)?', 'Pionaarr Mods ' + data['site'], js) if new != js: with open(ui, 'w', encoding='utf-8', newline='\n') as f: f.write(new) print('stamped builder/ui.js with Pionaarr Mods %s' % data['site']) if __name__ == '__main__': main()