Skip to content

Commit f07d528

Browse files
pirapiraclaude
pirapira
and
claude
committed
Add leanSpec module import coverage script
test_leanspec_imports.sh tries importing each of the 120 leanSpec source modules through the LeanPython interpreter and reports PASS/XFAIL/FAIL. Current results: 23 pass, 0 unexpected fail, 97 xfail (expected). Passing modules: all types/ (12), all ssz/ (5), koalabear, snappy/ (5). Expected failures are annotated with GitHub issue numbers: - #4 dict subclass inheritance - #5 lean_multisig_py / XMSS chain (pending pure Python impl) - #6 numpy/numba (pending pure Python impl) - #7 asyncio (chain/clock, networking, sync, validator, api, node) - #9 prometheus_client Also updates README.md with all three test commands. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 8b0041c commit f07d528

2 files changed

Lines changed: 210 additions & 2 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -54,10 +54,12 @@ echo 'print("Hello, World!")' > hello.py
5454
lake exe leanPython hello.py
5555
```
5656

57-
Run the test suite:
57+
Run the tests:
5858

5959
```bash
60-
lake test
60+
lake test # Lean unit tests (#eval/#guard assertions)
61+
./test_leanspec.sh # E2E integration tests (19 tiers against real leanSpec files)
62+
./test_leanspec_imports.sh # leanSpec module import coverage (PASS/XFAIL/FAIL)
6163
```
6264

6365
## Project structure

‎test_leanspec_imports.sh‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
#!/bin/bash
2+
# leanSpec module import coverage test.
3+
# Attempts to import each leanSpec source module through the LeanPython interpreter.
4+
# Reports PASS / XFAIL (expected failure with issue#) / FAIL (unexpected).
5+
#
6+
# Usage: ./test_leanspec_imports.sh
7+
8+
set -euo pipefail
9+
10+
EXE=".lake/build/bin/leanPython"
11+
SRC="/home/zksecurity/leanPython/references/leanSpec/src"
12+
13+
if [ ! -f "$EXE" ]; then
14+
echo "error: $EXE not found — run 'lake build' first"
15+
exit 1
16+
fi
17+
18+
PASS=0
19+
FAIL=0
20+
XFAIL=0
21+
FAILURES=""
22+
23+
# Expected failures: module_pattern -> "issue_number reason"
24+
# Modules matching these patterns are XFAIL (expected to fail).
25+
declare -A XFAIL_MAP
26+
27+
# #5: lean_multisig_py (pending pure Python impl)
28+
XFAIL_MAP["lean_spec.subspecs.xmss.aggregation"]="#5 lean_multisig_py"
29+
# Transitive: everything importing xmss.aggregation
30+
XFAIL_MAP["lean_spec.subspecs.xmss.interface"]="#5 imports aggregation transitively"
31+
XFAIL_MAP["lean_spec.subspecs.containers.attestation.attestation"]="#5 imports xmss.aggregation"
32+
XFAIL_MAP["lean_spec.subspecs.containers.block.block"]="#5 imports xmss.aggregation"
33+
XFAIL_MAP["lean_spec.subspecs.containers.block.types"]="#4,#5 dict subclass + xmss.aggregation"
34+
XFAIL_MAP["lean_spec.subspecs.containers.state.state"]="#5 imports xmss.aggregation"
35+
XFAIL_MAP["lean_spec.subspecs.containers.validator"]="#5 imports xmss.containers->config"
36+
XFAIL_MAP["lean_spec.subspecs.containers.attestation.aggregation_bits"]="#5 imports validator->xmss"
37+
XFAIL_MAP["lean_spec.subspecs.containers.state.types"]="#5 imports validator->xmss"
38+
39+
# #5: XMSS modules that need lean_spec.config (get_args / PEP 695 __value__)
40+
XFAIL_MAP["lean_spec.config"]="#5 get_args + PEP695 __value__"
41+
XFAIL_MAP["lean_spec.subspecs.xmss.constants"]="#5 imports lean_spec.config"
42+
XFAIL_MAP["lean_spec.subspecs.xmss.types"]="#5 imports xmss.constants"
43+
XFAIL_MAP["lean_spec.subspecs.xmss.containers"]="#5 imports xmss.constants"
44+
XFAIL_MAP["lean_spec.subspecs.xmss._validation"]="#5 imports lean_spec.config"
45+
XFAIL_MAP["lean_spec.subspecs.xmss.hypercube"]="#5 imports xmss.constants"
46+
XFAIL_MAP["lean_spec.subspecs.xmss.message_hash"]="#5 imports xmss chain"
47+
XFAIL_MAP["lean_spec.subspecs.xmss.poseidon"]="#5 imports xmss chain"
48+
XFAIL_MAP["lean_spec.subspecs.xmss.prf"]="#5 imports xmss.constants"
49+
XFAIL_MAP["lean_spec.subspecs.xmss.rand"]="#5 imports xmss.constants"
50+
XFAIL_MAP["lean_spec.subspecs.xmss.subtree"]="#5 imports xmss.types"
51+
XFAIL_MAP["lean_spec.subspecs.xmss.target_sum"]="#5 imports xmss.constants"
52+
XFAIL_MAP["lean_spec.subspecs.xmss.tweak_hash"]="#5 imports xmss chain"
53+
XFAIL_MAP["lean_spec.subspecs.xmss.utils"]="#5 imports xmss chain"
54+
55+
# #6: numpy/numba (pending pure Python impl)
56+
XFAIL_MAP["lean_spec.subspecs.poseidon2.permutation"]="#6 numpy"
57+
XFAIL_MAP["lean_spec.subspecs.poseidon2.constants"]="#6 imports permutation transitively"
58+
59+
# #7: asyncio
60+
XFAIL_MAP["lean_spec.__main__"]="#7 asyncio"
61+
XFAIL_MAP["lean_spec.subspecs.chain.clock"]="#7 asyncio"
62+
XFAIL_MAP["lean_spec.subspecs.chain.service"]="#7 asyncio"
63+
XFAIL_MAP["lean_spec.subspecs.api.server"]="#7 asyncio"
64+
XFAIL_MAP["lean_spec.subspecs.api.routes"]="#7 imports api.server"
65+
XFAIL_MAP["lean_spec.subspecs.api.endpoints.checkpoints"]="#7 imports api chain"
66+
XFAIL_MAP["lean_spec.subspecs.api.endpoints.fork_choice"]="#7 imports api chain"
67+
XFAIL_MAP["lean_spec.subspecs.api.endpoints.health"]="#7 imports api chain"
68+
XFAIL_MAP["lean_spec.subspecs.api.endpoints.metrics"]="#7 imports api chain"
69+
XFAIL_MAP["lean_spec.subspecs.api.endpoints.states"]="#7 asyncio"
70+
XFAIL_MAP["lean_spec.subspecs.networking.client.event_source"]="#7 asyncio"
71+
XFAIL_MAP["lean_spec.subspecs.networking.client.reqresp_client"]="#7 asyncio"
72+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.service"]="#7 asyncio"
73+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.transport"]="#7 asyncio + cryptography"
74+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.behavior"]="#7 asyncio"
75+
XFAIL_MAP["lean_spec.subspecs.networking.service.service"]="#7 asyncio"
76+
XFAIL_MAP["lean_spec.subspecs.networking.service.events"]="#7 imports service chain"
77+
XFAIL_MAP["lean_spec.subspecs.networking.transport.quic.connection"]="#7 asyncio + aioquic"
78+
XFAIL_MAP["lean_spec.subspecs.networking.transport.quic.stream_adapter"]="#7 asyncio"
79+
XFAIL_MAP["lean_spec.subspecs.node.node"]="#7 asyncio"
80+
XFAIL_MAP["lean_spec.subspecs.sync.service"]="#7 asyncio"
81+
XFAIL_MAP["lean_spec.subspecs.sync.backfill_sync"]="#7 imports sync chain"
82+
XFAIL_MAP["lean_spec.subspecs.sync.checkpoint_sync"]="#7 imports sync chain"
83+
XFAIL_MAP["lean_spec.subspecs.sync.head_sync"]="#7 imports sync chain"
84+
XFAIL_MAP["lean_spec.subspecs.validator.service"]="#7 asyncio"
85+
XFAIL_MAP["lean_spec.subspecs.validator.registry"]="#5,#7 imports xmss + chain"
86+
87+
# chain/__init__.py imports clock.py which uses asyncio, blocking all chain.* imports
88+
XFAIL_MAP["lean_spec.subspecs.chain.config"]="#7 chain/__init__.py imports clock->asyncio"
89+
XFAIL_MAP["lean_spec.subspecs.containers.checkpoint"]="#7 imports chain->clock->asyncio"
90+
XFAIL_MAP["lean_spec.subspecs.containers.config"]="#7 imports chain->clock->asyncio"
91+
XFAIL_MAP["lean_spec.subspecs.containers.slot"]="#7 imports chain->clock->asyncio"
92+
93+
# #9: prometheus_client
94+
XFAIL_MAP["lean_spec.subspecs.metrics.registry"]="#9 prometheus_client"
95+
XFAIL_MAP["lean_spec.subspecs.forkchoice.store"]="#5,#9 xmss + prometheus_client"
96+
97+
# Other external deps (cryptography, aioquic, sqlite3)
98+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.crypto"]="cryptography lib"
99+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.keys"]="cryptography lib"
100+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.handshake"]="cryptography lib"
101+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.codec"]="imports discovery chain"
102+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.config"]="imports discovery chain"
103+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.messages"]="imports discovery chain"
104+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.packet"]="imports discovery chain"
105+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.routing"]="imports discovery chain"
106+
XFAIL_MAP["lean_spec.subspecs.networking.discovery.session"]="imports discovery chain"
107+
XFAIL_MAP["lean_spec.subspecs.networking.enr.enr"]="cryptography lib"
108+
XFAIL_MAP["lean_spec.subspecs.networking.enr.eth2"]="imports enr chain"
109+
XFAIL_MAP["lean_spec.subspecs.networking.enr.keys"]="cryptography lib"
110+
XFAIL_MAP["lean_spec.subspecs.networking.transport.identity.keypair"]="cryptography lib"
111+
XFAIL_MAP["lean_spec.subspecs.networking.transport.identity.signature"]="cryptography lib"
112+
XFAIL_MAP["lean_spec.subspecs.networking.transport.peer_id"]="imports identity chain"
113+
XFAIL_MAP["lean_spec.subspecs.networking.transport.protocols"]="imports transport chain"
114+
XFAIL_MAP["lean_spec.subspecs.networking.transport.quic.tls"]="aioquic + cryptography"
115+
XFAIL_MAP["lean_spec.subspecs.networking.peer"]="imports networking chain"
116+
XFAIL_MAP["lean_spec.subspecs.networking.config"]="imports networking chain"
117+
XFAIL_MAP["lean_spec.subspecs.networking.types"]="imports networking chain"
118+
XFAIL_MAP["lean_spec.subspecs.networking.varint"]="imports networking chain"
119+
XFAIL_MAP["lean_spec.subspecs.networking.reqresp.codec"]="imports networking chain"
120+
XFAIL_MAP["lean_spec.subspecs.networking.reqresp.handler"]="imports networking chain"
121+
XFAIL_MAP["lean_spec.subspecs.networking.reqresp.message"]="imports networking chain"
122+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.mcache"]="imports networking chain"
123+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.mesh"]="imports networking chain"
124+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.message"]="imports networking chain"
125+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.parameters"]="imports networking chain"
126+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.rpc"]="imports networking chain"
127+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.topic"]="imports networking chain"
128+
XFAIL_MAP["lean_spec.subspecs.networking.gossipsub.types"]="imports networking chain"
129+
XFAIL_MAP["lean_spec.subspecs.storage.sqlite"]="sqlite3"
130+
XFAIL_MAP["lean_spec.subspecs.storage.database"]="imports storage chain"
131+
XFAIL_MAP["lean_spec.subspecs.storage.exceptions"]="imports storage chain"
132+
XFAIL_MAP["lean_spec.subspecs.storage.namespaces"]="imports storage chain"
133+
XFAIL_MAP["lean_spec.subspecs.sync.block_cache"]="imports sync chain"
134+
XFAIL_MAP["lean_spec.subspecs.sync.config"]="imports sync chain"
135+
XFAIL_MAP["lean_spec.subspecs.sync.peer_manager"]="imports sync chain"
136+
XFAIL_MAP["lean_spec.subspecs.sync.states"]="imports sync chain"
137+
XFAIL_MAP["lean_spec.subspecs.genesis.config"]="#5 imports xmss chain"
138+
XFAIL_MAP["lean_spec.subspecs.genesis.state"]="#5 imports genesis.config"
139+
140+
# --- Test runner ---
141+
142+
MAIN_PY="$SRC/_leanpython_import_test.py"
143+
cleanup() { rm -f "$MAIN_PY"; }
144+
trap cleanup EXIT
145+
146+
try_import() {
147+
local mod="$1"
148+
149+
cat > "$MAIN_PY" << PYEOF
150+
import $mod
151+
PYEOF
152+
153+
local output
154+
output=$(timeout 30 "$EXE" "$MAIN_PY" 2>&1) || true
155+
156+
if [ -z "$output" ]; then
157+
return 0 # success (no output = no error)
158+
else
159+
echo "$output"
160+
return 1
161+
fi
162+
}
163+
164+
echo "=== LeanPython leanSpec import coverage ==="
165+
echo ""
166+
167+
# Collect all modules
168+
MODULES=$(find "$SRC/lean_spec" -name "*.py" -not -name "__init__.py" -not -path "*__pycache__*" \
169+
| sed "s|$SRC/||; s|/|.|g; s|\.py$||" | sort)
170+
171+
for mod in $MODULES; do
172+
xfail_reason="${XFAIL_MAP[$mod]:-}"
173+
174+
output=$(try_import "$mod" 2>&1) && ok=true || ok=false
175+
176+
if $ok; then
177+
if [ -n "$xfail_reason" ]; then
178+
# Unexpectedly passed — still good, report as PASS
179+
printf " PASS %-60s (was xfail: %s)\n" "$mod" "$xfail_reason"
180+
else
181+
printf " PASS %s\n" "$mod"
182+
fi
183+
PASS=$((PASS + 1))
184+
else
185+
if [ -n "$xfail_reason" ]; then
186+
printf " XFAIL %-60s [%s]\n" "$mod" "$xfail_reason"
187+
XFAIL=$((XFAIL + 1))
188+
else
189+
printf " FAIL %s\n" "$mod"
190+
FAILURES="$FAILURES\n $mod: $(echo "$output" | head -1)"
191+
FAIL=$((FAIL + 1))
192+
fi
193+
fi
194+
done
195+
196+
echo ""
197+
echo "Results: $PASS pass, $FAIL fail, $XFAIL xfail (expected)"
198+
199+
if [ $FAIL -gt 0 ]; then
200+
echo ""
201+
echo "Unexpected failures:"
202+
printf "$FAILURES\n"
203+
exit 1
204+
fi
205+
206+
exit 0

0 commit comments

Comments
 (0)