Add Dart AOT depth (reference + 3 scripts); fix encoding rule and add P26
New reference: - dart-aot.md: pinning the Dart version, building a matching decompiler, the object-pool model and reference indexes, register/boolean conventions, the three assembly signatures that identify most business logic, the locating workflow, patch strategy, and the traps specific to this layer (including ambiguous identifiers that resolve to protocol fields). New scripts (parameterized, no machine-specific paths): - dart_pprefs.py: object-pool offset -> code-site index, decoded arithmetically. Avoids the full-capstone pass that costs minutes and 1-2 GB and reads as a hang. - dart_pool_strings.py: framed literal recovery with file offsets. Defaults to one-byte because automatic two-byte (UTF-16) detection is unreliable; --find searches both encodings and is the reliable way to locate a known literal. - dart_disasm.py: annotated windowed disassembly (pool + boolean annotations) and a B/BL caller index that needs no disassembler. Fixes and integration: - framework-runtimes.md: point the Dart depth at dart-aot.md, and make the string-encoding rule precise (ASCII is one-byte/UTF-8; CJK is UTF-16LE) instead of leaving it as 'try both'. - pitfalls.md: P26 -- self-built tooling that fails in a way that reads as a target finding (silent arithmetic, helper shadowing a stdlib module, measuring against an incomplete reference, and a knob that does not move the metric). - SKILL.md / README.md: index the new reference and scripts. Validated by running all three scripts against a real Dart AOT sample: the pool-ref index agrees with an independently built index to 99.2%, and the literal extractor's one-byte output was verified string-by-string against a decompiler's pool listing.
This commit is contained in:
inliver233
@@ -0,0 +1,212 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Annotated, windowed disassembly of a Dart AOT snapshot, plus a caller index.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
Two things make Dart AOT code readable at all:
|
||||
|
||||
1. **Pool annotations.** A raw `ldr x2, [x27, #0x9d0]` is meaningless; annotated with the
|
||||
pool entry it becomes `"vipflag"` and the surrounding code explains itself. This tool
|
||||
resolves those loads and prints what they point at, including how many other sites
|
||||
reference the same entry.
|
||||
2. **Windowed disassembly.** Decoding an entire multi-MB `.text` in one pass is how a host
|
||||
gets pinned at 1-2 GB and the run looks like a hang. Disassemble only the ranges you
|
||||
care about -- that is what this tool does, and it is not a limitation but the method.
|
||||
|
||||
Booleans are annotated too: Dart materialises `true`/`false` as small offsets from the null
|
||||
register, so `add x0, x22, #0x20` is marked TRUE and `#0x30` is marked FALSE. Those are the
|
||||
cheapest patch sites in the whole snapshot.
|
||||
|
||||
The caller index answers the other half of every question -- "what calls this function".
|
||||
It is built from B/BL target arithmetic on the raw words, so it needs no disassembler at all
|
||||
and finishes in seconds.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
# a window around one address (default: 0x180 before, 0x220 after)
|
||||
python dart_disasm.py libapp.so --pp pp.txt --refs pp_refs.json 0x26e390
|
||||
|
||||
# an explicit range
|
||||
python dart_disasm.py libapp.so --pp pp.txt --refs pp_refs.json --range 0x64f310 0x64f3e0
|
||||
|
||||
# widen / narrow the window
|
||||
python dart_disasm.py libapp.so --refs pp_refs.json 0x64f1bc --before 0x80 --after 0x40
|
||||
|
||||
# caller index
|
||||
python dart_disasm.py libapp.so --build-index callers.json
|
||||
python dart_disasm.py libapp.so --index callers.json 0x26e230 0x64f0f0
|
||||
|
||||
INPUTS
|
||||
------
|
||||
* `--pp` is the decompiler's pool listing (blutter `pp.txt`): lines shaped
|
||||
`[pp+0x9d0] String: "vipflag"`. Without it, loads are annotated with the raw offset only.
|
||||
* `--refs` is the pool-offset -> code-site index built by `dart_pprefs.py`.
|
||||
|
||||
Requires `capstone` for disassembly; the caller index works without it.
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
import sys
|
||||
|
||||
POOL_REG = 27
|
||||
BOOL_TRUE, BOOL_FALSE = 0x20, 0x30
|
||||
PP_LINE = re.compile(r'^\[(pp\+0x[0-9a-f]+)\](.*)$')
|
||||
|
||||
|
||||
def text_section(path):
|
||||
blob = open(path, 'rb').read()
|
||||
if blob[:4] != b'\x7fELF':
|
||||
raise SystemExit('not an ELF file: %s' % path)
|
||||
if blob[4] != 2:
|
||||
raise SystemExit('not ELF64: %s' % path)
|
||||
e_shoff = struct.unpack_from('<Q', blob, 0x28)[0]
|
||||
e_shentsize = struct.unpack_from('<H', blob, 0x3A)[0]
|
||||
e_shnum = struct.unpack_from('<H', blob, 0x3C)[0]
|
||||
e_shstrndx = struct.unpack_from('<H', blob, 0x3E)[0]
|
||||
|
||||
def sh(i):
|
||||
return struct.unpack_from('<IIQQQQ', blob, e_shoff + i * e_shentsize)
|
||||
|
||||
_, _, _, _, shstr_off, shstr_size = sh(e_shstrndx)
|
||||
strs = blob[shstr_off:shstr_off + shstr_size]
|
||||
for i in range(e_shnum):
|
||||
name, _t, _f, addr, off, size = sh(i)
|
||||
end = strs.find(b'\x00', name)
|
||||
if strs[name:end].decode('ascii', 'replace') == '.text':
|
||||
return addr, blob[off:off + size]
|
||||
raise SystemExit('.text not found')
|
||||
|
||||
|
||||
def load_pp(path):
|
||||
pool = {}
|
||||
if not path or not os.path.exists(path):
|
||||
return pool
|
||||
for line in open(path, encoding='utf-8', errors='replace'):
|
||||
m = PP_LINE.match(line.rstrip('\n'))
|
||||
if m:
|
||||
pool[int(m.group(1)[5:], 16)] = m.group(2).strip()[:120]
|
||||
return pool
|
||||
|
||||
|
||||
def load_refs(path):
|
||||
if not path or not os.path.exists(path):
|
||||
return {}
|
||||
return {int(k, 16): v for k, v in json.load(open(path, encoding='utf-8')).items()}
|
||||
|
||||
|
||||
def disasm_window(sample, addr, before, after, pp, refs):
|
||||
from capstone import Cs, CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN
|
||||
|
||||
text_addr, blob = text_section(sample)
|
||||
end_of_text = text_addr + len(blob)
|
||||
start = max(text_addr, addr - before)
|
||||
stop = min(end_of_text, addr + after)
|
||||
|
||||
md = Cs(CS_ARCH_ARM64, CS_MODE_LITTLE_ENDIAN)
|
||||
md.skipdata = True # without this, one undecodable byte silently ends the range
|
||||
md.detail = True # without this, operands are unavailable
|
||||
|
||||
print('range 0x%x..0x%x focus=0x%x' % (start, stop, addr))
|
||||
pending = {}
|
||||
for ins in md.disasm(blob[start - text_addr:stop - text_addr], start):
|
||||
if ins.id == 0: # SKIPDATA pseudo-instruction: no operands
|
||||
print('%x: %s' % (ins.address, ins.mnemonic))
|
||||
continue
|
||||
note = ''
|
||||
ops = ins.operands
|
||||
rn = ins.reg_name
|
||||
if (ins.mnemonic == 'add' and len(ops) == 3 and ops[1].type == 1
|
||||
and rn(ops[1].reg) == 'x27' and ops[2].type == 2
|
||||
and ops[2].shift.value == 12):
|
||||
pending[rn(ops[0].reg)] = ops[2].imm << 12
|
||||
elif (ins.mnemonic == 'add' and len(ops) == 3 and ops[1].type == 1
|
||||
and rn(ops[1].reg) == 'x22' and ops[2].type == 2):
|
||||
note = {BOOL_TRUE: ' ; TRUE', BOOL_FALSE: ' ; FALSE'}.get(ops[2].imm, '')
|
||||
elif ins.mnemonic in ('ldr', 'ldur') and len(ops) == 2 and ops[1].type == 3:
|
||||
base = rn(ops[1].mem.base) if ops[1].mem.base else ''
|
||||
off = None
|
||||
if base == 'x27':
|
||||
off = ops[1].mem.disp
|
||||
elif base in pending:
|
||||
off = pending.pop(base) + ops[1].mem.disp
|
||||
if off is not None:
|
||||
text = pp.get(off, '')
|
||||
sites = refs.get(off, [])
|
||||
note = ' ; pp+0x%x %s [%d refs]' % (off, text, len(sites))
|
||||
mark = ' <<<' if abs(ins.address - addr) < 4 else ''
|
||||
print('%x: %-8s %s%s%s' % (ins.address, ins.mnemonic, ins.op_str, note, mark))
|
||||
|
||||
|
||||
def build_index(sample, out):
|
||||
"""B/BL target -> call sites, decoded arithmetically (no disassembler needed)."""
|
||||
text_addr, blob = text_section(sample)
|
||||
index = {}
|
||||
words = struct.unpack_from('<%dI' % (len(blob) // 4), blob, 0)
|
||||
for i, w in enumerate(words):
|
||||
if w & 0xFC000000 in (0x14000000, 0x94000000):
|
||||
imm = w & 0x03FFFFFF
|
||||
if imm & 0x02000000:
|
||||
imm -= 0x04000000
|
||||
site = text_addr + i * 4
|
||||
index.setdefault(site + (imm << 2), []).append(site)
|
||||
with open(out, 'w', encoding='utf-8') as fh:
|
||||
json.dump({hex(k): v for k, v in index.items()}, fh)
|
||||
print('distinct targets : %d' % len(index))
|
||||
print('call sites : %d' % sum(len(v) for v in index.values()))
|
||||
print('written %s' % out)
|
||||
|
||||
|
||||
def lookup_index(sample, path, addrs):
|
||||
index = {int(k, 16): v for k, v in json.load(open(path, encoding='utf-8')).items()}
|
||||
for spec in addrs:
|
||||
target = int(spec, 16)
|
||||
sites = sorted(index.get(target, []))
|
||||
print('0x%x: %d caller(s)' % (target, len(sites)))
|
||||
for s in sites[:80]:
|
||||
print(' 0x%x' % s)
|
||||
if len(sites) > 80:
|
||||
print(' ... %d more' % (len(sites) - 80))
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(
|
||||
description=__doc__.split('\n')[0],
|
||||
formatter_class=argparse.RawDescriptionHelpFormatter, epilog=__doc__)
|
||||
ap.add_argument('sample', help='libapp.so')
|
||||
ap.add_argument('addr', nargs='*', help='hex address(es) to disassemble around')
|
||||
ap.add_argument('--pp', metavar='PP_TXT', help="decompiler pool listing (blutter pp.txt)")
|
||||
ap.add_argument('--refs', metavar='REFS_JSON', help='index from dart_pprefs.py')
|
||||
ap.add_argument('--range', nargs=2, metavar=('START', 'END'))
|
||||
ap.add_argument('--before', default='0x180')
|
||||
ap.add_argument('--after', default='0x220')
|
||||
ap.add_argument('--build-index', metavar='OUT_JSON')
|
||||
ap.add_argument('--index', metavar='CALLERS_JSON')
|
||||
a = ap.parse_args()
|
||||
|
||||
if a.build_index:
|
||||
return build_index(a.sample, a.build_index)
|
||||
if a.index:
|
||||
if not a.addr:
|
||||
raise SystemExit('give at least one hex address to look up')
|
||||
return lookup_index(a.sample, a.index, a.addr)
|
||||
|
||||
pp = load_pp(a.pp)
|
||||
refs = load_refs(a.refs)
|
||||
if a.range:
|
||||
start, stop = int(a.range[0], 16), int(a.range[1], 16)
|
||||
disasm_window(a.sample, start, 0, stop - start, pp, refs)
|
||||
return 0
|
||||
if not a.addr:
|
||||
raise SystemExit('give an address, --range, --build-index, or --index')
|
||||
for spec in a.addr:
|
||||
disasm_window(a.sample, int(spec, 16),
|
||||
int(a.before, 0), int(a.after, 0), pp, refs)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,273 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Recover string literals from a Dart AOT snapshot, and locate them by file offset.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
Strings are the cheapest anchor for locating logic in a stripped AOT snapshot: field
|
||||
names, endpoint paths and labels survive compilation even when every symbol name is gone.
|
||||
Two things this gives you that a decompiler's own listing does not:
|
||||
|
||||
* **file offsets**, where the constant physically sits in the binary (what you need to
|
||||
patch bytes, as opposed to a pool offset);
|
||||
* **identifiers the decompiler's listing omits** -- measured on a real sample, most of
|
||||
what this recovers was absent from the pool listing entirely. Treat the two as
|
||||
complementary, not competitive.
|
||||
|
||||
THE FRAMING
|
||||
-----------
|
||||
[ tag ][ payload ] tag = 0x80 | (len << 1) | <two-byte flag>
|
||||
|
||||
* tag bit 7 set -> a string entry
|
||||
* (tag >> 1) & 0x3F -> payload length in characters (this framing tops out at 63)
|
||||
* the low bit -> nominally "payload is UTF-16LE"
|
||||
|
||||
ENCODING, AND WHY ONE-BYTE IS THE DEFAULT
|
||||
-----------------------------------------
|
||||
The two payload kinds are searched differently, and this is where most people lose time:
|
||||
|
||||
* **one-byte** payloads are ASCII / Latin-1 -> a **UTF-8 byte search finds them**
|
||||
* **two-byte** payloads are **UTF-16LE** -> a UTF-8 search returns *zero hits*
|
||||
|
||||
So search ASCII as UTF-8 and CJK/non-Latin as UTF-16LE; forcing one encoding on both is
|
||||
the classic false negative (`pitfalls.md` P25).
|
||||
|
||||
**But automatic two-byte detection does not work reliably.** The low tag bit is not a
|
||||
dependable discriminator against ordinary data, and UTF-16LE decoding essentially never
|
||||
fails -- *any* byte pair is a legal code unit -- so decoding success filters nothing. On a
|
||||
real sample, two-byte output was ~10k entries of near-pure garbage that happened to form
|
||||
long runs, while one-byte output was almost entirely genuine. Hence:
|
||||
|
||||
* default: one-byte only (reliable; verified against a decompiler listing on a real sample)
|
||||
* `--two-byte`: opt in, and **verify every hit yourself**
|
||||
* to find a *known* non-Latin literal, use `--find` -- a direct search is far more
|
||||
reliable than enumerating candidates
|
||||
|
||||
WHY THE RUN LENGTH IS REPORTED
|
||||
------------------------------
|
||||
A per-byte scan for tag-like bytes hits constantly inside ordinary data; a multi-MB
|
||||
snapshot yields hundreds of thousands of candidates. Most real string tables are laid out
|
||||
consecutively, so a genuine entry usually begins a run of back-to-back entries (each
|
||||
entry's tag+payload ends exactly where the next begins). `--min-chain` uses that to shed
|
||||
the scattered tail.
|
||||
|
||||
Note what the measurement showed: raising this threshold barely changes one-byte
|
||||
precision while steadily destroying recall. It is a **noise filter, not a confidence
|
||||
score** -- do not tune it expecting accuracy to improve.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
python dart_pool_strings.py libapp.so strings.tsv
|
||||
python dart_pool_strings.py libapp.so strings.tsv --pp pp.txt # annotate cross-hits
|
||||
python dart_pool_strings.py libapp.so strings.tsv --min-chain 2 --two-byte
|
||||
python dart_pool_strings.py libapp.so --find /activateVipCode --find vipflag
|
||||
python dart_pool_strings.py libapp.so --stats
|
||||
|
||||
OUTPUT
|
||||
------
|
||||
offset_hex <TAB> text <TAB> run <TAB> form [<TAB> in_pp]
|
||||
|
||||
`in_pp` is present with `--pp`: `1` means the decompiler's own pool listing contains this
|
||||
exact string. Absence is **not** evidence of a false positive -- the decompiler's listing
|
||||
is incomplete by design.
|
||||
|
||||
NOTES
|
||||
-----
|
||||
* `--max` is capped at 63 by the framing; longer literals are serialized differently and
|
||||
will not appear here (a few hundred per sample at most).
|
||||
* Never edit a string without counting its references first: a shared entry changes every
|
||||
consumer at once (`references/dart-aot.md` section 10).
|
||||
"""
|
||||
import argparse
|
||||
import collections
|
||||
import re
|
||||
import sys
|
||||
|
||||
TAG_STR = 0x80
|
||||
BAD = re.compile(r'[\x00-\x08\x0b\x0c\x0e-\x1f]')
|
||||
|
||||
|
||||
def decode_payload(raw, two_byte):
|
||||
if two_byte:
|
||||
if len(raw) % 2:
|
||||
return None
|
||||
try:
|
||||
text = raw.decode('utf-16-le')
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
else:
|
||||
try:
|
||||
text = raw.decode('utf-8')
|
||||
except UnicodeDecodeError:
|
||||
return None
|
||||
if BAD.search(text):
|
||||
return None
|
||||
return text
|
||||
|
||||
|
||||
def extract(blob, min_len, max_len, want_two_byte):
|
||||
"""Return {start: (end, text, form)} for every framed candidate of the wanted kind."""
|
||||
n = len(blob)
|
||||
cands = {}
|
||||
for i in range(n - 2):
|
||||
tag = blob[i]
|
||||
if not (tag & TAG_STR):
|
||||
continue
|
||||
ln = (tag >> 1) & 0x3F
|
||||
if not (min_len <= ln <= max_len):
|
||||
continue
|
||||
two = tag & 1
|
||||
if two and not want_two_byte:
|
||||
continue
|
||||
end = i + 1 + (ln * 2 if two else ln)
|
||||
if end > n:
|
||||
continue
|
||||
text = decode_payload(blob[i + 1:end], two)
|
||||
if text is None or len(text.strip()) < min_len:
|
||||
continue
|
||||
cands[i] = (end, text, 'utf16le' if two else 'one-byte')
|
||||
return cands
|
||||
|
||||
|
||||
def run_lengths(cands):
|
||||
"""For each candidate, how many back-to-back entries follow it (inclusive)."""
|
||||
nxt = {s: cands[s][0] for s in cands}
|
||||
lengths = {}
|
||||
closed = set()
|
||||
for start in sorted(cands):
|
||||
if start in closed:
|
||||
continue
|
||||
seq = []
|
||||
cur = start
|
||||
guard = 0
|
||||
while cur in nxt and cur not in closed:
|
||||
seq.append(cur)
|
||||
closed.add(cur)
|
||||
cur = nxt[cur]
|
||||
guard += 1
|
||||
if guard > 1_000_000:
|
||||
break
|
||||
for i, item in enumerate(seq):
|
||||
lengths[item] = len(seq) - i
|
||||
return lengths
|
||||
|
||||
|
||||
def load_pp(path):
|
||||
"""Strings present in a decompiler pool listing, for cross-annotation."""
|
||||
if not path:
|
||||
return set()
|
||||
out = set()
|
||||
pat = re.compile(r'^\[pp\+0x[0-9a-f]+\] String: (.*)$')
|
||||
for line in open(path, encoding='utf-8', errors='replace'):
|
||||
m = pat.match(line.strip())
|
||||
if not m:
|
||||
continue
|
||||
s = m.group(1).strip()
|
||||
if len(s) >= 2 and s[0] == '"' and s[-1] == '"':
|
||||
s = s[1:-1]
|
||||
if s:
|
||||
out.add(s)
|
||||
return out
|
||||
|
||||
|
||||
def histogram(lengths):
|
||||
counts = collections.Counter(lengths.values())
|
||||
for run in sorted(counts)[:10]:
|
||||
print(' run=%-4d %d' % (run, counts[run]))
|
||||
tail = sum(v for k, v in counts.items() if k > 10)
|
||||
if tail:
|
||||
print(' run>%-3d %d' % (10, tail))
|
||||
|
||||
|
||||
def search(blob, needles):
|
||||
total = 0
|
||||
for needle in needles:
|
||||
for label, enc in (('utf-8', needle.encode('utf-8')),
|
||||
('utf-16le', needle.encode('utf-16-le'))):
|
||||
offsets = []
|
||||
start = blob.find(enc)
|
||||
while start != -1 and len(offsets) < 8:
|
||||
offsets.append(start)
|
||||
start = blob.find(enc, start + 1)
|
||||
if offsets:
|
||||
total += len(offsets)
|
||||
print('%-26s %-9s file offset %s' % (
|
||||
needle, label, ' '.join(hex(o) for o in offsets)))
|
||||
if not total:
|
||||
print('no hits in either encoding. Try the shortest distinctive fragment -- text is\n'
|
||||
'often assembled from parts, and a literal may only exist as fragments\n'
|
||||
'(pitfalls.md P25).')
|
||||
return 0
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
|
||||
ap.add_argument('sample', help='libapp.so or any artifact holding the snapshot')
|
||||
ap.add_argument('out', nargs='?', help='output tsv')
|
||||
ap.add_argument('--min', type=int, default=4, help='minimum payload length (default 4)')
|
||||
ap.add_argument('--max', type=int, default=63,
|
||||
help="maximum payload length; 63 is this framing's ceiling (default 63)")
|
||||
ap.add_argument('--min-chain', type=int, default=3,
|
||||
help='noise filter: minimum run of consecutive entries (default 3). '
|
||||
'This does not improve accuracy -- see the module docstring.')
|
||||
ap.add_argument('--two-byte', action='store_true',
|
||||
help='ALSO emit UTF-16 candidates. Detection is unreliable; verify each.')
|
||||
ap.add_argument('--pp', metavar='PP_TXT',
|
||||
help='decompiler pool listing, to mark which strings it also contains')
|
||||
ap.add_argument('--stats', action='store_true',
|
||||
help='print the run-length histogram and exit')
|
||||
ap.add_argument('--keep-isolated', action='store_true',
|
||||
help='also write entries below --min-chain (noisy; triage only)')
|
||||
ap.add_argument('--find', action='append', default=[], metavar='TEXT',
|
||||
help='search a literal in BOTH encodings and print file offsets '
|
||||
'(repeatable; the reliable way to locate a known literal)')
|
||||
a = ap.parse_args()
|
||||
|
||||
blob = open(a.sample, 'rb').read()
|
||||
|
||||
if a.find:
|
||||
return search(blob, a.find)
|
||||
if not a.out and not a.stats:
|
||||
raise SystemExit('give an output path, or --find / --stats')
|
||||
|
||||
cands = extract(blob, a.min, a.max, a.two_byte)
|
||||
lengths = run_lengths(cands)
|
||||
|
||||
if a.stats:
|
||||
print('candidates (two-byte included: %s): %d' % (a.two_byte, len(cands)))
|
||||
print('run-length histogram:')
|
||||
histogram(lengths)
|
||||
return 0
|
||||
|
||||
pp = load_pp(a.pp)
|
||||
kept = [s for s in cands if lengths[s] >= a.min_chain]
|
||||
rows = kept + (sorted(s for s in cands if lengths[s] < a.min_chain)
|
||||
if a.keep_isolated else [])
|
||||
with open(a.out, 'w', encoding='utf-8') as fh:
|
||||
fh.write('# offset_hex\ttext\trun\tform%s\n' % ('\tin_pp' if pp else ''))
|
||||
for s in sorted(rows):
|
||||
_end, text, form = cands[s]
|
||||
flat = text.replace('\n', '\\n').replace('\t', ' ')
|
||||
extra = ('\t%d' % (1 if text in pp else 0)) if pp else ''
|
||||
fh.write('0x%06x\t%s\t%d\t%s%s\n' % (s, flat, lengths[s], form, extra))
|
||||
|
||||
one = sum(1 for s in kept if cands[s][2] == 'one-byte')
|
||||
print('candidates : %d' % len(cands))
|
||||
print('kept (run >= %d) : %d (one-byte %d / utf16le %d)'
|
||||
% (a.min_chain, len(kept), one, len(kept) - one))
|
||||
print('below threshold : %d%s' % (
|
||||
len(cands) - len(kept), '' if a.keep_isolated else ' [--keep-isolated to keep them]'))
|
||||
if pp:
|
||||
hits = sum(1 for s in kept if cands[s][1] in pp)
|
||||
print('also in --pp listing : %d / %d (absence is NOT a false positive: the'
|
||||
% (hits, len(kept)))
|
||||
print(' decompiler listing is incomplete by design)')
|
||||
if a.two_byte:
|
||||
print('WARNING: --two-byte detection is unreliable (see module docstring);')
|
||||
print(' verify each UTF-16 hit before trusting it.')
|
||||
print('written %s' % a.out)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
# -*- coding: utf-8 -*-
|
||||
"""Dart AOT object-pool reference index: pool offset -> the instructions that load it.
|
||||
|
||||
WHY THIS EXISTS
|
||||
---------------
|
||||
blutter's `pp.txt` tells you what is *in* the object pool (strings, types, closures,
|
||||
field metadata) but not which code reads it. "Who uses this string" is the most frequent
|
||||
question in Dart AOT work, and it needs an index you build once and query constantly.
|
||||
|
||||
Building that index with a general-purpose disassembler is the trap here: decoding every
|
||||
instruction of a multi-MB `.text` with detail enabled and asking for per-instruction
|
||||
register writes costs minutes of CPU and 1-2 GB of peak memory, and on a 16 GB host it is
|
||||
indistinguishable from a hang -- no output, no progress, nothing to interrupt. Only three
|
||||
encodings can reach the pool, so this decodes them from the raw 32-bit words instead:
|
||||
seconds, and a flat memory profile.
|
||||
|
||||
Recognised forms (AArch64, little endian), with x27 = pool base:
|
||||
|
||||
add xD, x27, #imm12, lsl #12 -> xD = x27 + (imm12 << 12)
|
||||
ldr xD, [xN, #imm12 * 8] -> 64-bit or 32-bit variant
|
||||
ldur xD, [xN, #simm9] -> 64-bit or 32-bit variant
|
||||
|
||||
A reference is either a direct load whose base is x27, or a load consuming the register an
|
||||
ADD just derived from x27. The compiler emits that pair back to back, so a small window
|
||||
keeps it exact without full liveness analysis.
|
||||
|
||||
USAGE
|
||||
-----
|
||||
python dart_pprefs.py libapp.so pp_refs.json # build the index
|
||||
python dart_pprefs.py --lookup pp_refs.json 0x1d1a8 0xc9d0
|
||||
python dart_pprefs.py libapp.so out.json --window 5 # widen the ADD->LDR window
|
||||
|
||||
NOTES
|
||||
-----
|
||||
* Offsets are **pool** offsets -- the `pp+0x...` space used by pp.txt -- not file offsets.
|
||||
Keep the two spaces apart in your notes; conflating them costs hours.
|
||||
* One offset commonly has many reference sites. Count them before editing the entry:
|
||||
a shared constant will change behaviour everywhere at once (see references/dart-aot.md).
|
||||
"""
|
||||
import argparse
|
||||
import json
|
||||
import struct
|
||||
import sys
|
||||
|
||||
POOL_REG = 27 # x27: object-pool base in Dart AOT on arm64
|
||||
DEFAULT_WINDOW = 3 # instructions an ADD-derived address may survive
|
||||
|
||||
|
||||
def text_section(path):
|
||||
"""Return (vaddr, bytes) of .text, parsing ELF64 section headers directly."""
|
||||
blob = open(path, 'rb').read()
|
||||
if blob[:4] != b'\x7fELF':
|
||||
raise SystemExit('not an ELF file: %s' % path)
|
||||
if blob[4] != 2:
|
||||
raise SystemExit('not ELF64: %s' % path)
|
||||
e_shoff = struct.unpack_from('<Q', blob, 0x28)[0]
|
||||
e_shentsize = struct.unpack_from('<H', blob, 0x3A)[0]
|
||||
e_shnum = struct.unpack_from('<H', blob, 0x3C)[0]
|
||||
e_shstrndx = struct.unpack_from('<H', blob, 0x3E)[0]
|
||||
|
||||
def sh(i):
|
||||
return struct.unpack_from('<IIQQQQ', blob, e_shoff + i * e_shentsize)
|
||||
|
||||
_, _, _, _, shstr_off, shstr_size = sh(e_shstrndx)
|
||||
strs = blob[shstr_off:shstr_off + shstr_size]
|
||||
for i in range(e_shnum):
|
||||
name, _typ, _flags, addr, off, size = sh(i)
|
||||
end = strs.find(b'\x00', name)
|
||||
if strs[name:end].decode('ascii', 'replace') == '.text':
|
||||
return addr, blob[off:off + size]
|
||||
raise SystemExit('.text not found; pass --text with its virtual address')
|
||||
|
||||
|
||||
def sx(value, bits):
|
||||
m = 1 << (bits - 1)
|
||||
return (value ^ m) - m
|
||||
|
||||
|
||||
def scan(text_addr, blob, window):
|
||||
refs = {}
|
||||
pending = {}
|
||||
words = struct.unpack_from('<%dI' % (len(blob) // 4), blob, 0)
|
||||
|
||||
for i, w in enumerate(words):
|
||||
addr = text_addr + i * 4
|
||||
|
||||
# add xD, x27, #hi, lsl #12
|
||||
if w & 0xFFC00000 == 0x91400000 and (w >> 5) & 0x1F == POOL_REG:
|
||||
# Keep the shift out of the mask expression: `a & m << n` binds as
|
||||
# `a & (m << n)` in Python, which silently produces a wrong base.
|
||||
imm = (w >> 10) & 0xFFF
|
||||
pending[w & 0x1F] = [imm << 12, i]
|
||||
continue
|
||||
|
||||
ldr64 = w & 0xFFC00000 == 0xF9400000
|
||||
ldr32 = w & 0xFFC00000 == 0xB9400000
|
||||
ldur64 = w & 0xFFE00C00 == 0xF8400000
|
||||
ldur32 = w & 0xFFE00C00 == 0xB8400000
|
||||
|
||||
if ldr64 or ldr32 or ldur64 or ldur32:
|
||||
rn = (w >> 5) & 0x1F
|
||||
rd = w & 0x1F
|
||||
if ldr64 or ldr32:
|
||||
disp = ((w >> 10) & 0xFFF) << 3
|
||||
else:
|
||||
disp = sx((w >> 12) & 0x1FF, 9)
|
||||
if rn == POOL_REG:
|
||||
refs.setdefault(disp, []).append(addr)
|
||||
elif rn in pending:
|
||||
base, seq = pending.pop(rn)
|
||||
if i - seq <= window:
|
||||
refs.setdefault(base + disp, []).append(addr)
|
||||
pending.pop(rd, None) # this load overwrote rd
|
||||
continue
|
||||
|
||||
if w & 0xFFC00000 in (0x91000000, 0x91400000):
|
||||
pending.pop(w & 0x1F, None)
|
||||
for r in [r for r, (_o, s) in pending.items() if i - s > window]:
|
||||
del pending[r]
|
||||
|
||||
return refs
|
||||
|
||||
|
||||
def build(so, out, window):
|
||||
text_addr, blob = text_section(so)
|
||||
refs = scan(text_addr, blob, window)
|
||||
with open(out, 'w', encoding='utf-8') as fh:
|
||||
json.dump({hex(k): v for k, v in refs.items()}, fh)
|
||||
total = sum(len(v) for v in refs.values())
|
||||
print('.text vaddr=0x%x bytes=%d' % (text_addr, len(blob)))
|
||||
print('distinct pool offsets : %d' % len(refs))
|
||||
print('reference sites : %d' % total)
|
||||
for off in sorted(refs, key=lambda k: -len(refs[k]))[:8]:
|
||||
print(' pp+0x%-8x %4d sites' % (off, len(refs[off])))
|
||||
print('written %s' % out)
|
||||
|
||||
|
||||
def lookup(path, offsets):
|
||||
refs = {int(k, 16): v for k, v in json.load(open(path, encoding='utf-8')).items()}
|
||||
for spec in offsets:
|
||||
off = int(spec, 16)
|
||||
sites = sorted(refs.get(off, []))
|
||||
print('pp+0x%x: %d ref site(s)' % (off, len(sites)))
|
||||
for s in sites:
|
||||
print(' 0x%x' % s)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description=__doc__.split('\n')[0])
|
||||
ap.add_argument('sample', nargs='?', help='libapp.so (build mode)')
|
||||
ap.add_argument('out', nargs='?', help='output json (build mode)')
|
||||
ap.add_argument('--lookup', metavar='REFS_JSON', help='query an existing index')
|
||||
ap.add_argument('--window', type=int, default=DEFAULT_WINDOW,
|
||||
help='instructions an ADD-derived address may survive (default %d)'
|
||||
% DEFAULT_WINDOW)
|
||||
a = ap.parse_args()
|
||||
|
||||
if a.lookup:
|
||||
rest = sys.argv[sys.argv.index('--lookup') + 2:]
|
||||
if not rest:
|
||||
raise SystemExit('give at least one hex pool offset')
|
||||
lookup(a.lookup, rest)
|
||||
return 0
|
||||
if not (a.sample and a.out):
|
||||
raise SystemExit('usage: dart_pprefs.py <libapp.so> <out.json>')
|
||||
build(a.sample, a.out, a.window)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user