diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee20b73b..5d357c36 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -39,11 +39,11 @@ on: # The web installer + docs site are served from Pages by the deploy-pages job # below; a change to them must trigger a deploy or it never reaches the live site # (the eth-only-provisioning fix shipped a commit that didn't auto-deploy because - # web-installer was missing here). src/ui/install-picker*.js is already covered by src/**. + # mooninstaller was missing here). src/ui/install-picker*.js is already covered by src/**. # docs/** covers every page rendered into the Pages root by MkDocs; mkdocs.yml is # the site config (nav/theme) — a nav change with no doc change must still redeploy. - 'docs/**' - - 'web-installer/**' + - 'mooninstaller/**' - 'mkdocs.yml' workflow_dispatch: inputs: @@ -91,7 +91,7 @@ jobs: TAG: ${{ inputs.tag || github.ref_name }} run: uv run moondeck/ci/verify_version.py --tag "$TAG" - # The shipping firmware list, read from the generated web-installer/firmwares.json + # The shipping firmware list, read from the generated mooninstaller/firmwares.json # (projected from build_esp32.py's FIRMWARES dict, drift-guarded by # check_firmwares.py). Emitted as a JSON array so build-esp32's matrix can # fromJSON() it — GitHub matrices can't read a file at parse time, so a job @@ -108,7 +108,7 @@ jobs: - id: gen run: | set -euo pipefail - echo "list=$(jq -c '[.firmwares[] | select(.ships) | .name]' web-installer/firmwares.json)" >> "$GITHUB_OUTPUT" + echo "list=$(jq -c '[.firmwares[] | select(.ships) | .name]' mooninstaller/firmwares.json)" >> "$GITHUB_OUTPUT" build-esp32: needs: [verify-version, firmwares] @@ -225,6 +225,19 @@ jobs: # Per-firmware flasher_args.json — the release job feeds it to # generate_manifest.py so offsets come from the real build. cp "$B/flasher_args.json" "dist/flasher-${{ matrix.firmware }}.json" + # MoonBase, the second boot image on the 4 MB tables (built alongside those firmwares + # by build_esp32.py): chip-shared, so staged under a shared name like the partition + # table. The slot-0 otadata is what makes a fresh install boot the APP with MoonBase + # standing by: blank otadata would boot MoonBase. Their manifests reference both + # (generate_manifest.py), and install-picker.js excludes them from OTA offers. + for MB in build/moonbase-*/projectMM-moonbase.bin; do + [ -f "$MB" ] || continue + CHIP=$(basename "$(dirname "$MB")"); CHIP=${CHIP#moonbase-} + cp "$MB" "dist/shared-moonbase-$CHIP.bin" + uv run python -c "import sys; sys.path.insert(0,'moondeck/build'); \ + import build_esp32, pathlib; \ + pathlib.Path('dist/shared-ota-data-slot0.bin').write_bytes(build_esp32.otadata_slot0_bytes())" + done - uses: actions/upload-artifact@v4 with: @@ -270,8 +283,28 @@ jobs: set -euo pipefail V=$(uv run python moondeck/build/compute_version.py --tag "$TAG") echo "version=$V" >> "$GITHUB_OUTPUT" + # ccache: the macOS runner compiles the whole desktop tree from cold every run (the ESP32 + # jobs get a prebuilt IDF container; this one gets nothing), which is why it was the slowest + # job in the workflow at ~10 min. CMake picks the launcher up from the environment, so the + # build script stays unchanged. The key rotates per run and restores from the newest + # matching prefix, the standard save-always cache shape for a compiler cache. + - name: Install ccache + run: brew install ccache + - name: Restore ccache + uses: actions/cache@v4 + with: + path: ~/Library/Caches/ccache + key: ccache-macos-14-${{ github.sha }} + restore-keys: | + ccache-macos-14- - name: Build + package macOS arm64 + env: + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache run: uv run moondeck/ci/package_desktop.py --version "${{ steps.ver.outputs.version }}" + - name: ccache stats + if: always() + run: ccache --show-stats - uses: actions/upload-artifact@v4 with: name: desktop-macos @@ -457,9 +490,9 @@ jobs: # — no CORS). The Pages-relative manifests are generated in the # deploy-pages job, where the web installer (CORS-bound) consumes them. BASE="https://github.com/${REPO}/releases/download/$TAG" - # The shipping firmware list — the same web-installer/firmwares.json the + # The shipping firmware list — the same mooninstaller/firmwares.json the # build matrix reads, so manifests and builds can't drift. - for F in $(jq -r '.firmwares[] | select(.ships) | .name' web-installer/firmwares.json); do + for F in $(jq -r '.firmwares[] | select(.ships) | .name' mooninstaller/firmwares.json); do uv run python moondeck/build/generate_manifest.py \ --firmware "$F" \ --version "$V" \ @@ -532,7 +565,7 @@ jobs: # is a glob pattern, the action does NOT strip `#` as comment syntax. files: | dist/firmware-*.bin - dist/shared-ota-data.bin + dist/shared-*.bin dist/partition-table-*.bin dist/manifest-*.json dist/projectMM-*.tar.gz @@ -615,7 +648,7 @@ jobs: --dir "pages/install/releases/$T" \ --pattern 'firmware-*.bin' \ --pattern '*-ota-data.bin' \ - --pattern 'shared-ota-data.bin' \ + --pattern 'shared-*.bin' \ --pattern 'partition-table-*.bin' \ --pattern 'manifest-*.json' \ --pattern 'projectMM-*.tar.gz' \ @@ -638,9 +671,9 @@ jobs: # Install page + the shared install-picker module sit at the root. # Each release's binaries + manifests live under releases//. mkdir -p pages/install - cp -r web-installer/. pages/install/ + cp -r mooninstaller/. pages/install/ cp src/ui/install-picker.js pages/install/ - # The board-catalog / chip-detection half of the picker — web-installer + # The board-catalog / chip-detection half of the picker — mooninstaller # only (not embedded in firmware), imported by index.html. Must ship to # Pages alongside install-picker.js or the ES-module import 404s. cp src/ui/install-picker-boards.js pages/install/ @@ -654,7 +687,7 @@ jobs: mkdir -p pages/install/assets/deviceModels # rel is "assets/deviceModels/." (the path served from /install/); # the source file lives in docs/ (i.e. docs/assets/deviceModels/...). - jq -r '.[].image // empty' web-installer/deviceModels.json | while read -r rel; do + jq -r '.[].image // empty' mooninstaller/deviceModels.json | while read -r rel; do src="docs/$rel" [ -f "$src" ] && cp "$src" "pages/install/$rel" \ || echo "WARNING: deviceModels.json image not found: $src" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9ac425f9..45a6fa94 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ name: Test # New Python/JS unit suites land under test/python and test/js and run here. # Paths cover every input to the host-side tests: the Python/JS sources under test -# (scripts, web-installer), the test files themselves, AND the device-side C++ frame +# (scripts, mooninstaller), the test files themselves, AND the device-side C++ frame # contract (src/core/Improv*.h + the platform handler) — a wire-format change in the # firmware must run the cross-language golden-vector tests so it can't drift from the # Python/JS builders silently. pull_request gates every PR; push runs main only (a @@ -20,7 +20,7 @@ on: pull_request: paths: &test-paths - 'moondeck/**' - - 'web-installer/**' + - 'mooninstaller/**' - 'src/core/ImprovFrame.h' - 'src/core/ImprovOpReassembler.h' - 'src/platform/esp32/platform_esp32_improv.cpp' diff --git a/.gitignore b/.gitignore index f3829b9c..c26719dd 100644 --- a/.gitignore +++ b/.gitignore @@ -91,6 +91,14 @@ esp32/sdkconfig esp32/sdkconfig.old esp32/managed_components/ esp32/dependencies.lock + +# MoonBase is its own ESP-IDF project, so it grows the same build artefacts: the component +# manager's downloads and the generated sdkconfig. Only sdkconfig.defaults is ours to keep. +moonbase/sdkconfig +moonbase/sdkconfig.old +moonbase/managed_components/ +moonbase/dependencies.lock +moonbase/build/ esp32/monitor.log # Generated live-scenario baseline cache (regenerated by run_live_scenario.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 492a4b38..f7b4fbb1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -208,7 +208,7 @@ add_custom_target(ui_embed DEPENDS ${CMAKE_SOURCE_DIR}/src/ui/ui_embedded.h) add_dependencies(mm_core ui_embed build_info_gen) # Windows: give the exe its own icon, so it is recognizable in Explorer, the taskbar and the Start -# menu whether it was installed or just unzipped. Generated from the same web-installer/favicon.png +# menu whether it was installed or just unzipped. Generated from the same mooninstaller/favicon.png # the macOS .icns comes from, so the mark has one source rather than a checked-in binary per # platform. Nothing here runs on macOS or Linux, which carry their icon in the .app and the .deb. set(MM_WIN_RESOURCES "") @@ -220,14 +220,14 @@ if(WIN32) # `uv run "; + +// The application slot. From the factory partition esp_ota_get_next_update_partition returns the +// first OTA slot, which is the one we want and is never the one we are running from. +const esp_partition_t* appPartition() { + return esp_ota_get_next_update_partition(nullptr); +} + +// Write a firmware image pulled from `url` straight into the application slot. This is what makes +// an unattended install possible: point MoonBase at a release asset and it fetches it itself. +// True while any install is writing the app slot. Torn reads are harmless (same display-only +// pattern the app uses); the guard only has to stop a SECOND install from starting. +volatile bool installing_ = false; +// Set by POST /cancel; the install loops poll it and abort cleanly back to the page. The app +// slot is left half-written, exactly like a power cut: MoonBase stays the boot target until a +// later install completes. +volatile bool cancelRequested_ = false; + +bool installFromUrl(const char* url) { + esp_http_client_config_t http = {}; + http.url = url; + http.timeout_ms = 20000; + http.keep_alive_enable = true; + http.crt_bundle_attach = esp_crt_bundle_attach; // GitHub and friends are HTTPS + // A GitHub release asset 302-redirects to a signed URL whose Location header (plus a + // multi-KB content-security-policy on the redirect response) overflows the client's + // default 512-byte header buffer, failing the connection AFTER a clean TLS handshake. + // Same values as the app's http_fetch_to_ota (platform_esp32_ota.cpp). + http.disable_auto_redirect = false; + http.max_redirection_count = 10; + // Large receive chunks: fewer, larger flash writes per loop. Measured on the bench + // (classic ESP32, 40 MHz DIO flash): 4 KB chunks stream at ~46 KB/s, 16 KB at ~86, + // 32 KB roughly the same as 16 (write-bound from there); RAM is plentiful here. + http.buffer_size = 32768; + http.buffer_size_tx = 4096; + esp_https_ota_config_t ota = {}; + ota.http_config = &http; + // One bulk erase of the whole slot up front instead of a sector erase inlined with every + // 4 KB write: per-sector erases dominated the install at ~25 KB/s (identical over TLS and + // plain HTTP, so the wire was never the limit). The upfront erase costs a few seconds, + // "preparing the install" covers it. + ota.bulk_flash_erase = true; + + esp_https_ota_handle_t handle = nullptr; + esp_err_t beginErr = esp_https_ota_begin(&ota, &handle); + if (beginErr != ESP_OK) { + // Numeric on purpose: the error-name table is compiled out for size + // (ESP_ERR_TO_NAME_LOOKUP=n), so esp_err_to_name would say "UNKNOWN ERROR". + std::snprintf(status_, sizeof(status_), "error: cannot start the download (0x%x)", + static_cast(beginErr)); + return false; + } + esp_err_t err; + while ((err = esp_https_ota_perform(handle)) == ESP_ERR_HTTPS_OTA_IN_PROGRESS) { + if (cancelRequested_) { + esp_https_ota_abort(handle); + std::snprintf(status_, sizeof(status_), "canceled"); + return false; + } + std::snprintf(status_, sizeof(status_), "downloading: %d of %d bytes", + esp_https_ota_get_image_len_read(handle), + esp_https_ota_get_image_size(handle)); + } + if (err != ESP_OK) { + esp_https_ota_abort(handle); // finish() is for a COMPLETE download; abort frees this one + std::snprintf(status_, sizeof(status_), "error: the download failed (0x%x)", + static_cast(err)); + return false; + } + if (esp_https_ota_finish(handle) != ESP_OK) { + std::snprintf(status_, sizeof(status_), "error: the image is not valid firmware"); + return false; + } + std::snprintf(status_, sizeof(status_), "installed, restarting"); + return true; +} + + +// --------------------------------------------------------------------------------------------- +// The HTTP server +// --------------------------------------------------------------------------------------------- +// +// Hand-written on raw sockets rather than esp_http_server: MoonBase serves one page and receives +// one file, and the component would cost more than the handlers do. One connection at a time is +// the right model here, since installing firmware is exclusive by nature. + +constexpr size_t kRecvChunk = 4096; + +void sendAll(int sock, const char* data, size_t len) { + size_t sent = 0; + while (sent < len) { + const int n = ::send(sock, data + sent, len - sent, 0); + if (n <= 0) return; // peer gone: the caller is finishing anyway + sent += static_cast(n); + } +} + +// The embedded logo (EMBED_FILES in CMakeLists; symbol names derive from the filename). +extern const uint8_t logoStart[] asm("_binary_moonlight_logo_png_start"); +extern const uint8_t logoEnd[] asm("_binary_moonlight_logo_png_end"); + +void sendBinary(int sock, const char* type, const uint8_t* data, size_t len) { + char head[192]; + const int n = std::snprintf(head, sizeof(head), + "HTTP/1.1 200 OK\r\nContent-Type: %s\r\nContent-Length: %u\r\n" + "Cache-Control: no-store\r\nConnection: close\r\n\r\n", + type, static_cast(len)); + if (n > 0) sendAll(sock, head, static_cast(n)); + sendAll(sock, reinterpret_cast(data), len); +} + +void sendResponse(int sock, const char* status, const char* type, const char* body) { + // no-store on everything: this address serves TWO different UIs over time (the app's and + // this one), and a browser that re-serves a cached copy of either shows a dead page. + char head[192]; + const int n = std::snprintf(head, sizeof(head), + "HTTP/1.1 %s\r\nContent-Type: %s\r\nContent-Length: %u\r\n" + "Cache-Control: no-store\r\nConnection: close\r\n\r\n", + status, type, static_cast(std::strlen(body))); + if (n > 0) sendAll(sock, head, static_cast(n)); + sendAll(sock, body, std::strlen(body)); +} + +// Write `contentLen` bytes from the socket into the application slot. `prefix` carries whatever +// arrived in the same read as the headers. +bool installFromSocketLocked(int sock, const char* prefix, size_t prefixLen, size_t contentLen) { + const esp_partition_t* part = appPartition(); + if (!part) { std::snprintf(status_, sizeof(status_), "error: no app partition"); return false; } + if (contentLen == 0 || contentLen > part->size) { + std::snprintf(status_, sizeof(status_), "error: image is %u bytes, the slot holds %u", + static_cast(contentLen), static_cast(part->size)); + return false; + } + + esp_ota_handle_t handle = 0; + if (esp_ota_begin(part, contentLen, &handle) != ESP_OK) { + std::snprintf(status_, sizeof(status_), "error: cannot start the install"); + return false; + } + + size_t written = 0; + if (prefixLen > contentLen) prefixLen = contentLen; // never store bytes past the declared body + if (prefixLen) { + if (esp_ota_write(handle, prefix, prefixLen) != ESP_OK) { + esp_ota_abort(handle); + std::snprintf(status_, sizeof(status_), "error: write failed"); + return false; + } + written = prefixLen; + } + + char* buf = static_cast(std::malloc(kRecvChunk)); + if (!buf) { esp_ota_abort(handle); std::snprintf(status_, sizeof(status_), "error: out of memory"); return false; } + while (written < contentLen) { + const size_t want = (contentLen - written) < kRecvChunk ? (contentLen - written) : kRecvChunk; + const int n = ::recv(sock, buf, want, 0); + if (n <= 0) break; // the upload was cut short + if (esp_ota_write(handle, buf, static_cast(n)) != ESP_OK) { + std::free(buf); + esp_ota_abort(handle); + std::snprintf(status_, sizeof(status_), "error: write failed"); + return false; + } + written += static_cast(n); + } + std::free(buf); + + if (written != contentLen) { + esp_ota_abort(handle); + std::snprintf(status_, sizeof(status_), "error: upload ended early (%u of %u bytes)", + static_cast(written), static_cast(contentLen)); + return false; + } + // esp_ota_end validates the image (magic and checksum) before we ever point the bootloader at + // it, which is what makes a power cut mid-write safe: otadata still names MoonBase. + if (esp_ota_end(handle) != ESP_OK) { + std::snprintf(status_, sizeof(status_), "error: the image is not valid firmware"); + return false; + } + if (esp_ota_set_boot_partition(part) != ESP_OK) { + std::snprintf(status_, sizeof(status_), "error: cannot set the boot partition"); + return false; + } + std::snprintf(status_, sizeof(status_), "installed, restarting"); + return true; +} + +// Read the request head, dispatch, and (on a successful install) restart into the application. +// The staged-URL install, off the main task (which serves meanwhile). A connect attempted +// straight after GOT_IP can fail (0x7002, ESP_ERR_HTTP_CONNECT) where the same connect succeeds +// seconds later: the LAN is still warming up around a freshly associated station. A short retry +// absorbs that; a genuinely unreachable URL still fails through to the page after the last +// attempt, where status_ shows the error. +char stagedUrlTask_[256]; + +void unattendedInstallTask(void*) { + // Remember the source across reboots (key "last_url", page prefill only): the retry + // escape must survive a power cycle, not just this session. + nvs_handle_t nh; + if (nvs_open("moonbase", NVS_READWRITE, &nh) == ESP_OK) { + nvs_set_str(nh, "last_url", stagedUrlTask_); + nvs_commit(nh); + nvs_close(nh); + } + for (int attempt = 0; attempt < 3 && !cancelRequested_; attempt++) { + if (attempt) vTaskDelay(pdMS_TO_TICKS(3000)); + if (installFromUrl(stagedUrlTask_)) esp_restart(); // straight back into the new app + // A failed attempt leaves its error in status_; while retries remain that error is + // TRANSIENT, and a watcher treating "error:" as terminal (the app's overlay does) + // must not see it. The final attempt's error stays as the terminal answer. + if (attempt < 2 && !cancelRequested_) + std::snprintf(status_, sizeof(status_), "download failed, retrying"); + } + cancelRequested_ = false; + installing_ = false; // set by the spawner; held across the retries + vTaskDelete(nullptr); +} + +void serveOne(int sock) { + // TCP does not coalesce: the header block (or a small body) can arrive in several + // segments, so read until the blank line is seen, bounded by the buffer. A request + // whose headers do not fit 1023 bytes is not one of ours and falls out as 404. + char head[1024]; + size_t got = 0; + const char* bodyStart = nullptr; + while (got < sizeof(head) - 1) { + const int n = ::recv(sock, head + got, sizeof(head) - 1 - got, 0); + if (n <= 0) break; + got += static_cast(n); + head[got] = '\0'; + if ((bodyStart = std::strstr(head, "\r\n\r\n"))) break; + } + if (got == 0) { ::close(sock); return; } // serveOne owns the fd; a bare return leaks it + head[got] = '\0'; + const size_t headLen = bodyStart ? static_cast(bodyStart + 4 - head) : got; + size_t prefixLen = got - headLen; + + // HTTP header names are case-insensitive; strcasestr is not in the std namespace but is + // provided by newlib, and the probe is bounded by the header buffer. + size_t contentLen = 0; + if (const char* cl = strcasestr(head, "Content-Length:")) { + contentLen = static_cast(std::strtoul(cl + 15, nullptr, 10)); + } + + bool installed = false; + if (std::strncmp(head, "POST /install-url", 17) == 0 && installing_) { + sendResponse(sock, "409 Conflict", "text/plain", "error: an install is already running"); + } else if (std::strncmp(head, "POST /install-url", 17) == 0) { + // The body is the URL itself; small enough to finish reading into the same buffer. + while (prefixLen < contentLen && headLen + prefixLen < sizeof(head) - 1) { + const int n = ::recv(sock, head + headLen + prefixLen, + sizeof(head) - 1 - headLen - prefixLen, 0); + if (n <= 0) break; + prefixLen += static_cast(n); + } + if (contentLen >= sizeof(stagedUrlTask_)) { + // Same 255-byte contract the app's route enforces (platform.h): refusing beats + // truncating into a URL that fails later as a misleading download error. + sendResponse(sock, "400 Bad Request", "text/plain", "error: url too long (max 255)"); + } else { + std::memcpy(stagedUrlTask_, head + headLen, prefixLen); + stagedUrlTask_[prefixLen] = '\0'; + std::snprintf(status_, sizeof(status_), "starting the install"); + cancelRequested_ = false; // a /cancel racing the previous task's exit must not latch + installing_ = true; // cleared by the task after its final attempt + if (xTaskCreate(unattendedInstallTask, "mb_install", 12288, nullptr, 5, nullptr) != pdPASS) { + // A failed spawn with the flag left set would refuse every later install: THE + // deadlock this guard exists to prevent. + installing_ = false; + std::snprintf(status_, sizeof(status_), "error: cannot start the install task"); + sendResponse(sock, "500 Internal Server Error", "text/plain", status_); + } else { + // 202: the install runs on its own task while this server keeps answering GET + // /moonbase with live progress; the caller watches that, not this response. + sendResponse(sock, "202 Accepted", "text/plain", status_); + } + } + } else if (std::strncmp(head, "POST /install", 13) == 0) { + if (installing_) { + sendResponse(sock, "409 Conflict", "text/plain", "error: an install is already running"); + } else { + installing_ = true; + installed = installFromSocketLocked(sock, head + headLen, prefixLen, contentLen); + installing_ = false; + sendResponse(sock, installed ? "200 OK" : "500 Internal Server Error", "text/plain", status_); + } + } else if (std::strncmp(head, "POST /boot-app", 14) == 0 && installing_) { + // Booting away mid-write would abandon a half-written slot; refuse, visibly. + sendResponse(sock, "409 Conflict", "text/plain", "error: an install is already running"); + } else if (std::strncmp(head, "POST /boot-app", 14) == 0) { + // Switch back to the installed application without installing anything. + // esp_ota_set_boot_partition validates the image first, so a half-written app is + // refused and the device stays here: only a bootable app can be booted. + const esp_partition_t* app = appPartition(); + const bool ok = app && esp_ota_set_boot_partition(app) == ESP_OK; + if (ok) std::snprintf(status_, sizeof(status_), "booting the app"); + else std::snprintf(status_, sizeof(status_), "error: no valid app image"); + sendResponse(sock, ok ? "200 OK" : "500 Internal Server Error", "text/plain", status_); + installed = ok; // reuse the reply-then-restart tail below + } else if (std::strncmp(head, "GET /logo.png", 13) == 0) { + sendBinary(sock, "image/png", logoStart, static_cast(logoEnd - logoStart)); + } else if (std::strncmp(head, "GET /last-url", 13) == 0) { + // The most recent install source, RAM-held: the page prefills its URL field with it, + // so Install doubles as retry, the escape after a cancel wiped the app slot. Empty + // after a power cycle. + sendResponse(sock, "200 OK", "text/plain", stagedUrlTask_); + } else if (std::strncmp(head, "POST /cancel", 12) == 0) { + // Cancel a running URL install: its loop polls the flag and aborts back to this page. + // (An upload cancels by dropping the connection; this server is busy receiving it.) + // Nothing to cancel is not an error worth a scary status, just say so. + if (installing_) { + cancelRequested_ = true; + sendResponse(sock, "200 OK", "text/plain", "canceling"); + } else { + sendResponse(sock, "200 OK", "text/plain", "nothing to cancel"); + } + } else if (std::strncmp(head, "GET /moonbase", 13) == 0) { + // Identity probe: the app UI polls this across the update cycle to tell which image is + // answering at the shared address (the app 404s it). Body = the live install status, so + // the poll doubles as a progress read during an unattended install. + sendResponse(sock, "200 OK", "text/plain", status_); + } else if (std::strncmp(head, "GET / ", 6) == 0 || std::strncmp(head, "GET /?", 6) == 0 || + std::strncmp(head, "GET /index", 10) == 0) { + // "/?" is the app page's cache-busting handoff to this page (app.js). + sendResponse(sock, "200 OK", "text/html", kPage); + } else { + sendResponse(sock, "404 Not Found", "text/plain", "not found"); + } + + ::shutdown(sock, SHUT_RDWR); + ::close(sock); + if (installed) { + // Let the reply reach the browser before the device goes away. + vTaskDelay(pdMS_TO_TICKS(500)); + esp_restart(); + } +} + +void serveForever() { + const int listener = ::socket(AF_INET, SOCK_STREAM, IPPROTO_TCP); + if (listener < 0) return; + int yes = 1; + ::setsockopt(listener, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + sockaddr_in addr = {}; + addr.sin_family = AF_INET; + addr.sin_addr.s_addr = htonl(INADDR_ANY); + addr.sin_port = htons(kHttpPort); + if (::bind(listener, reinterpret_cast(&addr), sizeof(addr)) != 0) { ::close(listener); return; } + if (::listen(listener, 1) != 0) { ::close(listener); return; } + + while (true) { + const int sock = ::accept(listener, nullptr, nullptr); + if (sock < 0) continue; + // A stalled peer must not hold MoonBase forever: the whole point is that the device stays + // reachable for the next attempt. + timeval tv = {}; + tv.tv_sec = 30; + ::setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &tv, sizeof(tv)); + ::setsockopt(sock, SOL_SOCKET, SO_SNDTIMEO, &tv, sizeof(tv)); + serveOne(sock); + } +} + +} // namespace + +extern "C" void app_main() { + esp_err_t nvs = nvs_flash_init(); + if (nvs == ESP_ERR_NVS_NO_FREE_PAGES || nvs == ESP_ERR_NVS_NEW_VERSION_FOUND) { + nvs_flash_erase(); + nvs_flash_init(); + } + netEvents_ = xEventGroupCreate(); + esp_netif_init(); + esp_event_loop_create_default(); + esp_event_handler_instance_register(IP_EVENT, ESP_EVENT_ANY_ID, &onGotIp, nullptr, nullptr); + + loadCredentials(); + + // The cascade: Ethernet where the config wires it (its DHCP window overlaps the WiFi + // join since the GOT_IP bit is shared), then WiFi STA with the stored credentials, then + // the open access point: the guarantee that a board is never unreachable because its + // credentials went stale. + // The unattended handoff: the app may have staged an install URL in NVS before rebooting + // into MoonBase (platform::moonbaseStageInstallUrl). Read AND erase it unconditionally, + // before anything can fail: a URL that crashes or fails can then never boot-loop the + // device, and a stale URL can never survive a failed network join to hijack a later, + // unrelated visit to MoonBase (one try per staging, ever). + char stagedUrl[256] = {}; + { + nvs_handle_t h; + if (nvs_open("moonbase", NVS_READWRITE, &h) == ESP_OK) { + size_t len = sizeof(stagedUrl); + if (nvs_get_str(h, "url", stagedUrl, &len) != ESP_OK) stagedUrl[0] = '\0'; + nvs_erase_key(h, "url"); + nvs_commit(h); + nvs_close(h); + } + } + + // With nothing staged, prefill the retry buffer from the remembered last source so the + // page offers it after any reboot. Never auto-installed: only the page's Install uses it. + if (!stagedUrl[0]) { + nvs_handle_t h; + if (nvs_open("moonbase", NVS_READONLY, &h) == ESP_OK) { + size_t len = sizeof(stagedUrlTask_); + if (nvs_get_str(h, "last_url", stagedUrlTask_, &len) != ESP_OK) stagedUrlTask_[0] = '\0'; + nvs_close(h); + } + } + + // ONE interface at a time, in the app's own preference order (eth where configured, else + // WiFi, else the AP): the app runs a single interface, so the browser is on that + // interface's address, and mirroring the preference is what keeps the address valid + // across the handoff without a second lease to confuse anyone. + bool online = false; + if (ethStart()) { + online = (xEventGroupWaitBits(netEvents_, kNetGotIp, pdFALSE, pdFALSE, + pdMS_TO_TICKS(8000)) & kNetGotIp) != 0; + if (!online) { + ethStop(); // no link or no lease: WiFi takes over, alone + // A lease that raced in between the wait timing out and the teardown is an + // interface that no longer exists; it must not satisfy the WiFi wait below. + xEventGroupClearBits(netEvents_, kNetGotIp); + } + } + if (!online) online = wifiStation(20000); + + // STA only: on the fallback AP the URL's network is not reachable, and a user is present. + // The install runs on its OWN task so the main task serves throughout: GET /moonbase then + // reports "downloading: N of M bytes" live, which is what the app's update overlay renders + // as a progress bar. 12 KB stack for the same reason as the main task: the TLS handshake. + if (online && stagedUrl[0]) { + // Status set BEFORE the task spawns: the overlay polls from the moment MoonBase + // answers, and "idle" would read as nothing happening while an install is pending. + std::snprintf(status_, sizeof(status_), "preparing the install"); + std::snprintf(stagedUrlTask_, sizeof(stagedUrlTask_), "%s", stagedUrl); + cancelRequested_ = false; // a /cancel racing a previous task's exit must not latch + installing_ = true; // cleared by the task after its final attempt + if (xTaskCreate(unattendedInstallTask, "mb_install", 12288, nullptr, 5, nullptr) != pdPASS) { + installing_ = false; // a latched flag would refuse every later install + std::snprintf(status_, sizeof(status_), "error: cannot start the install task"); + } + } + + if (!online) online = wifiAccessPoint(); + + // With no network there is nothing MoonBase can do but wait: a user who cannot reach it will + // reflash over USB, and restarting into an application slot that may be empty helps nobody. + if (online) serveForever(); + while (true) vTaskDelay(pdMS_TO_TICKS(1000)); +} diff --git a/moonbase/sdkconfig.defaults b/moonbase/sdkconfig.defaults new file mode 100644 index 00000000..08c0ab55 --- /dev/null +++ b/moonbase/sdkconfig.defaults @@ -0,0 +1,64 @@ +# MoonBase's size budget lives here. Measured on the ESP32 classic with WiFi + HTTP + OTA: IDF +# defaults gave 881 KB, and these settings bring the same functionality to ~590 KB. Nearly 300 KB +# of the saving is configuration rather than code, which is why this file is part of the design +# and not an afterthought. + +# Size over speed. MoonBase runs for a minute at a time and does one blocking download; there is +# no hot path to protect. (-O2 -> -Os: 69 KB) +CONFIG_COMPILER_OPTIMIZATION_SIZE=y + +# Link-time optimisation: cross-module dead-code elimination, the last big lever and the one +# MycilaSafeBoot also pulls. Costs build time, not runtime. +CONFIG_COMPILER_OPTIMIZATION_LTO=y + +# No log strings, no esp_err_to_name table, no assertion text, no console. Together ~117 KB. +# The cost is real: a misbehaving MoonBase says nothing over serial. It is accepted because the +# image is small enough to reason about whole, and because a board that cannot boot MoonBase is a +# USB recovery either way. +CONFIG_LOG_DEFAULT_LEVEL_NONE=y +CONFIG_LOG_MAXIMUM_LEVEL_NONE=y +CONFIG_ESP_ERR_TO_NAME_LOOKUP=n +CONFIG_COMPILER_OPTIMIZATION_ASSERTIONS_DISABLE=y +CONFIG_ESP_SYSTEM_PANIC_PRINT_HALT=y +CONFIG_ESP_CONSOLE_NONE=y +CONFIG_VFS_SUPPORT_TERMIOS=n + +# The C library's full printf is for humans reading logs, which MoonBase has none of. (~40 KB) +CONFIG_LIBC_NEWLIB_NANO_FORMAT=y + +# Network features this image will never use. IPv6 (~25 KB), WPA3/enterprise (~25 KB). +# SoftAP is deliberately KEPT (~35 KB): without it a board with wrong WiFi credentials can only be +# recovered over USB, which is exactly the situation MoonBase exists to avoid. +CONFIG_LWIP_IPV6=n +CONFIG_ESP_WIFI_ENABLE_WPA3_SAE=n +CONFIG_ESP_WIFI_ENTERPRISE_SUPPORT=n + +# One connection at a time, small buffers: MoonBase serves one page and receives one file. +CONFIG_LWIP_MAX_SOCKETS=6 +# RX sizing is the install's throughput ceiling (TCP window / RTT): the earlier minimal +# values capped a LAN download at ~25 KB/s. MoonBase's one job is moving a ~2 MB image, and +# with no app there is RAM to spare, so receive gets real buffers and a real window. +CONFIG_ESP_WIFI_STATIC_RX_BUFFER_NUM=8 +CONFIG_ESP_WIFI_DYNAMIC_RX_BUFFER_NUM=24 +CONFIG_LWIP_TCP_WND_DEFAULT=32768 +CONFIG_LWIP_TCP_RECVMBOX_SIZE=24 +CONFIG_ESP_WIFI_TX_BUFFER_TYPE=0 +CONFIG_ESP_WIFI_STATIC_TX_BUFFER_NUM=8 + +# The SAME table the application builds against, so the two images provably agree on where +# everything lives. MoonBase reads the filesystem (for the stored WiFi credentials) and never +# writes it, so a half-finished install cannot corrupt user config. +CONFIG_ESPTOOLPY_FLASHSIZE_4MB=y +CONFIG_PARTITION_TABLE_CUSTOM=y +CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="../esp32/partitions/esp32dev_moonbase.csv" + +# Plain-HTTP install URLs allowed alongside HTTPS: GitHub releases stay TLS-verified via the cert +# bundle, but a LAN source (MoonDeck serving a dev build) has no certificate and is inside the +# user's own network. The staged-URL handoff writes whatever the app was asked to install. +CONFIG_ESP_HTTPS_OTA_ALLOW_HTTP=y + +# The whole of MoonBase runs on the main task (app_main installs, then serves), and a GitHub +# install rides mbedTLS: certificate-chain verification against the bundle peaks near 8 KB of +# stack, which overflowed the 3.5 KB default the moment the TLS handshake ran (the plain-HTTP +# LAN path fit, which is why the bench missed it). RAM is not scarce here; 12 KB buys margin. +CONFIG_ESP_MAIN_TASK_STACK_SIZE=12288 diff --git a/moondeck/MoonDeck.md b/moondeck/MoonDeck.md index dcb8f21f..8d3dcd3a 100644 --- a/moondeck/MoonDeck.md +++ b/moondeck/MoonDeck.md @@ -18,7 +18,7 @@ Below: the UI behaviours common to every card, described once, then one section - **Tab persistence** — selected tab survives page refresh. - **Process detection** — on page load, checks if projectMM or idf.py is already running and shows Stop button. - **Network bar** (top of the sidebar): switch between known networks. Each network holds its own device list, last-used serial port, and WiFi credentials (consumed by Improv). On startup, MoonDeck auto-selects the network whose subnet matches the host's current LAN — moving the laptop between networks usually requires no clicks. Manual override (the dropdown) pins the selection until the pinned network's subnet stops matching the host. Add / Rename buttons next to the dropdown manage the catalog. State persisted in `moondeck/moondeck.json` under `networks` + `active_network`. -- **Device-model picker** on each device row: dropdown of device models from [web-installer/deviceModels.json](../web-installer/deviceModels.json) — the same catalog the web installer uses. When the device's firmware uniquely identifies one deviceModel (e.g. `esp32-eth` → Olimex Gateway), MoonDeck auto-deduces and mirrors the value to the device's `deviceModel` control on [SystemModule](../docs/moonmodules/core/SystemModule.md) via `POST /api/control` on next discover. For firmwares with no unique deviceModel (`esp32` runs on multiple), the user picks; MoonDeck pushes that value too. A device-reported deviceModel not in the catalog still shows up as ` (unknown)` so the value survives. MoonDeck's picker is a **text dropdown for an already-running device** — distinct from the web installer's flash-time *picture* deviceModel picker; both read the same catalog, but MoonDeck doesn't need the per-deviceModel `image`/`url` fields (those are installer-picker UX). Selecting a deviceModel pushes its full catalog config — each entry is a list of `{type, id, parent_id?, controls?}` module units (the [nested catalog schema](../web-installer/README.md), add-then-configure), so MoonDeck adds the deviceModel's modules (`POST /api/modules`) then sets their controls (`POST /api/control`); see `_push_device` in [moondeck.py](moondeck.py). +- **Device-model picker** on each device row: dropdown of device models from [mooninstaller/deviceModels.json](../mooninstaller/deviceModels.json) — the same catalog the web installer uses. When the device's firmware uniquely identifies one deviceModel (e.g. `esp32-eth` → Olimex Gateway), MoonDeck auto-deduces and mirrors the value to the device's `deviceModel` control on [SystemModule](../docs/moonmodules/core/SystemModule.md) via `POST /api/control` on next discover. For firmwares with no unique deviceModel (`esp32` runs on multiple), the user picks; MoonDeck pushes that value too. A device-reported deviceModel not in the catalog still shows up as ` (unknown)` so the value survives. MoonDeck's picker is a **text dropdown for an already-running device** — distinct from the web installer's flash-time *picture* deviceModel picker; both read the same catalog, but MoonDeck doesn't need the per-deviceModel `image`/`url` fields (those are installer-picker UX). Selecting a deviceModel pushes its full catalog config — each entry is a list of `{type, id, parent_id?, controls?}` module units (the [nested catalog schema](../mooninstaller/README.md), add-then-configure), so MoonDeck adds the deviceModel's modules (`POST /api/modules`) then sets their controls (`POST /api/control`); see `_push_device` in [moondeck.py](moondeck.py). ## Desktop Tab @@ -72,7 +72,7 @@ While the app is running, MoonDeck shows the button as **Stop** (a 5-second poll ![Installer2](../docs/assets/ui/installer2.png) ![Installer3](../docs/assets/ui/installer3.png) -Locally preview the web installer page at without tagging a release. Stages `web-installer/index.html` + `src/ui/install-picker.js` into `build/install-preview/` and serves them via Python's `http.server` on port 8421. +Locally preview the web installer page at without tagging a release. Stages `mooninstaller/index.html` + `src/ui/install-picker.js` into `build/install-preview/` and serves them via Python's `http.server` on port 8421. ```bash uv run moondeck/run/preview_installer.py @@ -81,7 +81,7 @@ uv run moondeck/run/preview_installer.py Long-running — MoonDeck shows **Stop** while the server is up. Two modes, picked automatically: -- **Render-only.** When no `build/esp32-*/projectMM.bin` is present, the picker populates against the real GitHub Releases API and dropdowns work, but clicking **Install** fails because the local server has no `releases/` tree. Useful for iterating on HTML / CSS / JS without burning a build. Equivalent to "Recipe A" in [web-installer/README.md](../web-installer/README.md). +- **Render-only.** When no `build/esp32-*/projectMM.bin` is present, the picker populates against the real GitHub Releases API and dropdowns work, but clicking **Install** fails because the local server has no `releases/` tree. Useful for iterating on HTML / CSS / JS without burning a build. Equivalent to "Recipe A" in [mooninstaller/README.md](../mooninstaller/README.md). - **Flash-ready.** When at least one ESP32 build exists, the script additionally stages every `build/esp32-*/projectMM.bin` it finds into `releases/local-dev/` and generates matching Pages-relative manifests via the same `generate_manifest.py` the release workflow uses. The picker shows `local-dev` as the newest tag; clicking **Install** flashes a USB-connected ESP32 and hands off to the repository's custom orchestrator UI (Improv-Serial provisioning + SET_DEVICE_MODEL + control fan-out, all in `install-orchestrator.js` — not ESP Web Tools). End-to-end, same code paths as the public installer. This is the developer's test ground for the install flow before deploying to GitHub Pages: Web Serial works on `http://localhost` without the secure-origin requirement that gates the public site. Add `?nocache=1` to the URL to bypass the picker's 5-minute sessionStorage cache while editing. @@ -149,7 +149,7 @@ Each gate carries an objective trigger read from the changed-file set, so a docs ### check_devices -Validate the installer device-model catalog (`web-installer/deviceModels.json`). +Validate the installer device-model catalog (`mooninstaller/deviceModels.json`). ```bash uv run moondeck/check/check_devices.py @@ -159,7 +159,7 @@ Checks each entry's required fields, that `firmwares` is a non-empty list, every ### check_firmwares -Verify the firmware projection (`web-installer/firmwares.json`) matches the `FIRMWARES` source. +Verify the firmware projection (`mooninstaller/firmwares.json`) matches the `FIRMWARES` source. ```bash uv run moondeck/check/check_firmwares.py @@ -1104,7 +1104,7 @@ Exit codes: `0` = all checks passed, `1` = device-side failure (probe or provisi - [src/core/ImprovFrame.h](../src/core/ImprovFrame.h) — the on-device parser - [src/platform/esp32/platform_esp32_improv.cpp](../src/platform/esp32/platform_esp32_improv.cpp) — the UART listener task -- [web-installer/index.html](../web-installer/index.html) — the web installer page +- [mooninstaller/index.html](../mooninstaller/index.html) — the web installer page - [src/ui/install-picker.js](../src/ui/install-picker.js) — the picker driving the install flow - [moondeck/build/improv_*.py](build/) — the host-side framing helpers diff --git a/moondeck/build/build_esp32.py b/moondeck/build/build_esp32.py index e3806a2e..3545ac36 100644 --- a/moondeck/build/build_esp32.py +++ b/moondeck/build/build_esp32.py @@ -124,7 +124,7 @@ def check_idf_pin(idf_path: Path) -> None: # `ships`: True for variants the release matrix builds + publishes. A variant can # exist here (buildable from the CLI) yet be held out of CI with ships=False. # This dict is the SINGLE source of truth — generate_firmwares.py projects it to -# web-installer/firmwares.json, which the CI matrix, the ESP Web Tools manifest +# mooninstaller/firmwares.json, which the CI matrix, the ESP Web Tools manifest # loops, and MoonDeck all read (check_firmwares.py guards the projection). FIRMWARES: dict[str, dict] = { # Default classic ESP32: WiFi AND Ethernet in one binary. The RMII Ethernet @@ -135,7 +135,8 @@ def check_idf_pin(idf_path: Path) -> None: # replaces the old separate `esp32` (WiFi-only) + `esp32-eth-wifi` keys. "esp32": { "chip": "esp32", - "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.eth"], + "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.eth", "sdkconfig.defaults.moonbase-4mb"], + "moonbase": True, # 4 MB: factory MoonBase + one big app slot (see moonbase/) "eth_only": False, "description": "ESP32 classic — WiFi + Ethernet (RMII; per-board pins/PHY " "from deviceModels.json, default LAN8720 pins).", @@ -150,12 +151,13 @@ def check_idf_pin(idf_path: Path) -> None: # API and web UI reachable through a forwarded host port. "qemu": { "chip": "esp32", - "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.qemu"], + "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.qemu", "sdkconfig.defaults.moonbase-4mb"], + "moonbase": True, # 4 MB: factory MoonBase + one big app slot (see moonbase/) "eth_only": True, "description": "ESP32 classic under QEMU, emulated Ethernet (openeth), no WiFi. " "Run with moondeck/qemu/run_qemu.py, not flashed to hardware.", "ships": False, - # Not silicon: keep it out of web-installer/firmwares.json entirely. `ships` already stops + # Not silicon: keep it out of mooninstaller/firmwares.json entirely. `ships` already stops # the release pipeline building it; this stops it reaching the installer's list, whose # entries are things a user can flash to a board. "installable": False, @@ -173,7 +175,8 @@ def check_idf_pin(idf_path: Path) -> None: "esp32-wrover": { "chip": "esp32", "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.eth", - "sdkconfig.defaults.wrover"], + "sdkconfig.defaults.wrover", "sdkconfig.defaults.moonbase-4mb"], + "moonbase": True, # 4 MB: factory MoonBase + one big app slot (see moonbase/) "eth_only": False, "description": "ESP32-WROVER (classic ESP32, 4 MB flash + 4 MB quad PSRAM) — WiFi + " "Ethernet. Same silicon as `esp32`; this variant enables PSRAM for " @@ -182,7 +185,8 @@ def check_idf_pin(idf_path: Path) -> None: }, "esp32-eth": { "chip": "esp32", - "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.eth"], + "fragments": ["sdkconfig.defaults", "sdkconfig.defaults.eth", "sdkconfig.defaults.moonbase-4mb"], + "moonbase": True, # 4 MB: factory MoonBase + one big app slot (see moonbase/) "eth_only": True, "description": "ESP32 classic — Ethernet only (WiFi compiled out; smaller " "image, more RAM). Per-board pins/PHY from deviceModels.json. The " @@ -605,6 +609,36 @@ def stale_feature_cache(build_dir: Path, extra: list[str], chip: str) -> str | N return (f"IDF_TARGET cached as {cached_target!r} but this firmware " f"wants {chip!r}") # The feature toggles whose presence/absence changes which code compiles. + # The FRAGMENT LIST is a feature flag too: IDF generates sdkconfig from + # SDKCONFIG_DEFAULTS only when the file is absent, so adding a fragment to a + # firmware (the MoonBase partition table did this first) silently leaves an + # existing dir on the OLD config. The cache still holds the LAST run's list at + # this point, so a mismatch is detectable and means: wipe and reconfigure. + wanted_frags = next((a.split("=", 1)[1] for a in extra + if a.startswith("-DSDKCONFIG_DEFAULTS=")), None) + m = re.search(r"^SDKCONFIG_DEFAULTS:[^=]*=(.*)$", text, re.MULTILINE) + cached_frags = m.group(1).strip() if m else None + if wanted_frags and cached_frags and cached_frags != wanted_frags: + return (f"SDKCONFIG_DEFAULTS cached as {cached_frags!r} but this firmware " + f"wants {wanted_frags!r}") + + # And the one generated value dangerous enough to verify outright: the partition table. The + # list comparison above cannot catch a dir poisoned BEFORE the rule existed (its cache already + # matches), so read what the fragments want (last fragment naming a table wins, IDF's own + # merge order) and compare against what the generated sdkconfig actually says. + wanted_table = None + if wanted_frags: + resolved = table_from_fragments(wanted_frags.split(";")) + wanted_table = str(resolved.relative_to(ESP32_DIR)) + gen = build_dir / "sdkconfig" + if wanted_table and gen.exists(): + m2 = re.search(r'^CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="([^"]+)"', + gen.read_text(), re.MULTILINE) + have_table = m2.group(1) if m2 else None + if have_table != wanted_table: + return (f"generated sdkconfig uses partition table {have_table!r} but the " + f"fragments want {wanted_table!r}") + # For each, "wanted" = does this firmware pass the -D, "cached" = is it set # in the existing cache. A disagreement means a stale dir. # MM_TASK_CPU_STATS is here too: toggling --task-cpu-stats on an existing dir must wipe, or the @@ -759,6 +793,130 @@ def main(): # Show flash/RAM usage summary subprocess.run(cmd + b_arg + ["size"], cwd=ESP32_DIR, env=env) + if FIRMWARES[firmware].get("moonbase"): + build_moonbase(cmd, env, chip) + + +# ---- MoonBase flash layout, shared by every consumer of the build output ---- +# flash_esp32.py (serial flash), generate_manifest.py (web installer), preview_installer.py +# (release preview) and run_qemu.py (emulator image) all assemble a flash layout from IDF's +# flasher_args.json. On a MoonBase table that file is WRONG about the app: IDF stages the app +# binary at the first app partition (0x10000: the factory slot, MoonBase's home), because it +# knows nothing about the two-image scheme. These helpers are the one place that knows better. + +def table_from_fragments(fragments) -> Path: + """The partition CSV a fragment list selects (last fragment naming one wins, IDF's own + merge order). The one resolver: moonbase_table_csv and stale_feature_cache both use it.""" + csv = ESP32_DIR / "partitions" / "esp32dev.csv" + for frag in fragments: + fp = ESP32_DIR / frag + if not fp.exists(): + continue + m = re.search(r'^CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="([^"]+)"', + fp.read_text(), re.MULTILINE) + if m: + csv = ESP32_DIR / m.group(1) + return csv + + +def moonbase_table_csv(firmware: str) -> Path: + return table_from_fragments(FIRMWARES[firmware]["fragments"]) + + +def partition_offsets(csv_path: Path) -> dict: + """SubType -> offset (hex string) for the rows a MoonBase layout needs: 'factory', + 'ota_0' and 'ota' (the otadata bookkeeping partition).""" + import csv as _csv + out = {} + for row in _csv.reader(csv_path.read_text().splitlines()): + if not row or row[0].strip().startswith("#") or len(row) < 5: + continue + subtype = row[2].strip() + if subtype in ("factory", "ota_0", "ota"): + out[subtype] = row[3].strip() + return out + + +def otadata_slot0_bytes() -> bytes: + """An otadata image with slot 0 (ota_0) selected, so a fresh full flash boots the app with + MoonBase standing by (blank otadata boots the factory slot: MoonBase). Two 4 KB copies; the + record is 32 bytes: uint32 seq, 24 bytes 0xFF, uint32 CRC over the seq alone + (bootloader_common_ota_select_crc: crc32_le seeded UINT32_MAX == zlib.crc32(seq, 0xFFFFFFFF)). + Byte-identical to what IDF's otatool writes for --slot 0, verified against a bench readback. + """ + import struct, zlib + seq = struct.pack(" list[tuple[str, Path]]: + """The corrected (offset, file) write list for a MoonBase-table flash: IDF's flash_files with + the app remapped to ota_0, the blank otadata replaced by the slot-0 image (written into the + build dir), and MoonBase added at the factory slot.""" + import json as _json + offs = partition_offsets(moonbase_table_csv(firmware)) + chip = FIRMWARES[firmware]["chip"] + moonbase_bin = build_dir.parent / f"moonbase-{chip}" / "projectMM-moonbase.bin" + if not all(k in offs for k in ("factory", "ota_0", "ota")) or not moonbase_bin.exists(): + raise FileNotFoundError( + f"MoonBase layout needs factory/ota_0/otadata offsets and a built image " + f"(run build_esp32.py first; missing: {moonbase_bin})") + otadata = build_dir / "ota_data_slot0.bin" + otadata.write_bytes(otadata_slot0_bytes()) + fa = _json.loads((build_dir / "flasher_args.json").read_text()) + writes: list[tuple[str, Path]] = [] + for off, rel in fa["flash_files"].items(): + name = Path(rel).name + if name == "projectMM.bin": + writes.append((offs["ota_0"], build_dir / rel)) + elif name == "ota_data_initial.bin": + writes.append((offs["ota"], otadata)) + else: + writes.append((off, build_dir / rel)) + writes.append((offs["factory"], moonbase_bin)) + return writes + + +def build_moonbase(cmd: list[str], env: dict, chip: str) -> None: + """Build the MoonBase image for `chip` into build/moonbase-. + + MoonBase (moonbase/) is the second boot image the 4 MB variants carry in their factory + partition: a small firmware whose job is installing the application, since a board with one + app slot cannot rewrite the partition it is executing from. It is chip-specific but variant- + agnostic, so the four classic variants share one build. Its size budget lives in + moonbase/sdkconfig.defaults; the shared partition table keeps the two images provably agreed + on where everything lives. + """ + moonbase_dir = ROOT / "moonbase" + build_dir = ROOT / "build" / f"moonbase-{chip}" + b_arg = ["-B", str(build_dir), f"-DSDKCONFIG={build_dir}/sdkconfig"] + # Same trap as stale_feature_cache: IDF generates sdkconfig from the defaults only when it is + # absent, so an edited moonbase/sdkconfig.defaults silently changes nothing. One defaults file + # here, so mtime is a sufficient staleness signal. + gen = build_dir / "sdkconfig" + defaults = moonbase_dir / "sdkconfig.defaults" + if gen.exists() and defaults.stat().st_mtime > gen.stat().st_mtime: + print(f"MoonBase build dir {build_dir.name} predates sdkconfig.defaults; " + "removing it for a clean reconfigure.") + shutil.rmtree(build_dir) + if not build_dir.exists(): + print(f"Setting MoonBase target to {chip}...") + r = subprocess.run(cmd + b_arg + ["set-target", chip], cwd=moonbase_dir, env=env) + if r.returncode != 0: + sys.exit(r.returncode) + print(f"Building MoonBase for {chip}...") + r = subprocess.run(cmd + b_arg + ["build"], cwd=moonbase_dir, env=env) + if r.returncode != 0: + sys.exit(r.returncode) + binp = build_dir / "projectMM-moonbase.bin" + if binp.exists(): + kb = binp.stat().st_size / 1024 + # Slot fit is printed by IDF itself ("Smallest app partition ... free"); repeating a + # hardcoded slot size here would lie the day the table changes. + print(f"MoonBase image: {kb:.0f} KB") + if __name__ == "__main__": main() diff --git a/moondeck/build/flash_esp32.py b/moondeck/build/flash_esp32.py index 2ee6b7bb..a975d0d6 100644 --- a/moondeck/build/flash_esp32.py +++ b/moondeck/build/flash_esp32.py @@ -11,7 +11,9 @@ """ import argparse +import json import re +import shutil import subprocess import sys import time @@ -24,7 +26,7 @@ from build_esp32 import find_idf, idf_env, idf_cmd, FIRMWARES, build_dir_for -CATALOG = ROOT / "web-installer" / "deviceModels.json" +CATALOG = ROOT / "mooninstaller" / "deviceModels.json" # MoonDeck / CLI flashing defaults FAST: this path is the DIY bench, where the operator # knows their board and the USB bridge is almost always a modern one that sustains 921600 # (~2x faster). A board with a flaky bridge opts DOWN via its catalog `flashBaud` (e.g. a @@ -79,6 +81,28 @@ def _fmt_age(seconds: float) -> str: return f"{s // 86400}d" +def _moonbase_flash_cmd(build_dir, firmware: str, port: str, baud: int, env: dict) -> list[str]: + """The explicit write list for a MoonBase-layout flash, moonbase_flash_files() is the one + place that knows the corrected layout (app remapped to ota_0, slot-0 otadata so the fresh + flash boots the app, MoonBase at factory). idf.py flash cannot be used here: IDF's own + flash_args stages the app at the factory offset.""" + from build_esp32 import moonbase_flash_files + try: + writes = moonbase_flash_files(firmware, build_dir) + except FileNotFoundError as e: + print(f"MoonBase flash: {e}") + sys.exit(1) + chip_m = re.search(r'CONFIG_IDF_TARGET="([^"]+)"', (build_dir / "sdkconfig").read_text()) + if not chip_m: + print(f"MoonBase flash: no CONFIG_IDF_TARGET in {build_dir / 'sdkconfig'}; rebuild first.") + sys.exit(1) + py = shutil.which("python", path=env.get("PATH", "")) or sys.executable + cmd = [py, "-m", "esptool", "--chip", chip_m.group(1), "--port", port, "--baud", str(baud), + "write_flash"] + for off, path in writes: + cmd += [off, str(path)] + return cmd + def main(): parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) parser.add_argument("--port", required=True, help="Serial port") @@ -155,7 +179,15 @@ def main(): # the console; we just also scan it. print(f"==> flash baud: {baud}") mac = "" - proc = subprocess.Popen(cmd + b_arg + ["flash", "-p", args.port, "-b", str(baud)], + # A MoonBase variant cannot use `idf.py flash`: with a factory + ota_0 table, IDF stages the + # application at the FACTORY offset (0x10000), which is MoonBase's slot and too small for it. + # The parts are placed at explicit offsets instead, the same shape the mooninstaller manifest + # uses, with each offset read from the built partition table rather than hardcoded. + if FIRMWARES.get(args.firmware, {}).get("moonbase"): + flash_cmd = _moonbase_flash_cmd(build_dir, args.firmware, args.port, baud, env) + else: + flash_cmd = cmd + b_arg + ["flash", "-p", args.port, "-b", str(baud)] + proc = subprocess.Popen(flash_cmd, cwd=ESP32_DIR, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) for line in proc.stdout: diff --git a/moondeck/build/generate_firmwares.py b/moondeck/build/generate_firmwares.py index 9c5c0bfc..bd642aeb 100644 --- a/moondeck/build/generate_firmwares.py +++ b/moondeck/build/generate_firmwares.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate web-installer/firmwares.json from the FIRMWARES dict. +"""Generate mooninstaller/firmwares.json from the FIRMWARES dict. The firmware-variant list was hand-copied across the CI matrix, the two ESP Web Tools manifest loops, MoonDeck, and the docs — six copies that drifted (MoonDeck's @@ -17,7 +17,7 @@ physical hardware). See docs/architecture.md § Firmware vs board. Inputs: - --out — firmwares.json destination (web-installer/firmwares.json). + --out — firmwares.json destination (mooninstaller/firmwares.json). """ import argparse diff --git a/moondeck/build/generate_manifest.py b/moondeck/build/generate_manifest.py index 2c7ae552..46e8a197 100644 --- a/moondeck/build/generate_manifest.py +++ b/moondeck/build/generate_manifest.py @@ -131,6 +131,25 @@ def main() -> int: print(f"generate_manifest: no recognised parts in {args.flasher_args}") return 1 + # MoonBase firmwares: flasher_args stages the app at the first app partition: the factory + # slot, MoonBase's home: because IDF knows nothing about the two-image scheme. Remap the app + # to ota_0, swap the blank ota-data for the slot-0 image (a blank one would boot MoonBase on + # first start), and add MoonBase at the factory slot. Offsets come from the same partition CSV + # the firmware builds with (moonbase_table_csv), so manifest and table cannot disagree. + # release.yml stages the two shared-moonbase assets these paths point at. + from build_esp32 import moonbase_table_csv, partition_offsets + spec = FIRMWARES[args.firmware] + if spec.get("moonbase"): + offs = partition_offsets(moonbase_table_csv(args.firmware)) + for part in parts: + if part["path"] == f"{prefix}.bin": + part["offset"] = int(offs["ota_0"], 16) + elif part["path"] == "shared-ota-data.bin": + part["path"] = "shared-ota-data-slot0.bin" + parts.append({"path": f"shared-moonbase-{spec['chip']}.bin", + "offset": int(offs["factory"], 16)}) + parts.sort(key=lambda p: p["offset"]) + # ESP Web Tools resolves the per-part `path` relative to the manifest URL, # so absolute URLs are the simplest robust shape — the manifest stays # correct whether it's served from Pages, a release page, or a mirror. diff --git a/moondeck/build/improv_provision.py b/moondeck/build/improv_provision.py index 1af9377e..92f3defa 100644 --- a/moondeck/build/improv_provision.py +++ b/moondeck/build/improv_provision.py @@ -220,7 +220,7 @@ def main() -> int: ap.add_argument("--timeout", type=float, default=45.0, help="Max seconds to wait for a final response (default: 45)") ap.add_argument("--device-model", dest="device_model", default=None, metavar="NAME", - help="deviceModel name from web-installer/deviceModels.json (e.g. " + help="deviceModel name from mooninstaller/deviceModels.json (e.g. " "'ESP32-S3 N16R8 Dev'). Resolves the deviceModel's TX-power cap " "(controls.Network.txPowerSetting) automatically and " "pushes the name via SET_DEVICE_MODEL after " @@ -230,7 +230,7 @@ def main() -> int: help="Send the SET_TX_POWER vendor RPC (0..21 whole dBm) " "BEFORE the credentials. Required for boards whose LDO " "browns out at full TX power (weak-powered boards → 8, see " - "web-installer/deviceModels.json) — without it the very first " + "mooninstaller/deviceModels.json) — without it the very first " "association fails and the cap can never arrive over HTTP.") args = ap.parse_args() @@ -279,7 +279,7 @@ def main() -> int: # same file the web installer and MoonDeck read. import json from pathlib import Path - boards_file = Path(__file__).resolve().parents[2] / "web-installer" / "deviceModels.json" + boards_file = Path(__file__).resolve().parents[2] / "mooninstaller" / "deviceModels.json" try: catalog = json.loads(boards_file.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError) as e: diff --git a/moondeck/build/improv_smoke_test.py b/moondeck/build/improv_smoke_test.py index 8c9f38c0..5005df5b 100755 --- a/moondeck/build/improv_smoke_test.py +++ b/moondeck/build/improv_smoke_test.py @@ -28,7 +28,7 @@ Recommended developer test before any commit touching: - src/core/ImprovFrame.h - src/platform/esp32/platform_esp32_improv.cpp - - web-installer/index.html + - mooninstaller/index.html - src/ui/install-picker.js - moondeck/build/improv_*.py diff --git a/moondeck/check/check_devices.py b/moondeck/check/check_devices.py index baaa3eeb..098bbdfa 100644 --- a/moondeck/check/check_devices.py +++ b/moondeck/check/check_devices.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Validate the installer board catalog (web-installer/deviceModels.json). +"""Validate the installer board catalog (mooninstaller/deviceModels.json). The catalog is hand-maintained data consumed identically by three clients (the web installer, the device UI's ?deviceModel= inject, and MoonDeck), so a typo drifts @@ -27,7 +27,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent -CATALOG = ROOT / "web-installer" / "deviceModels.json" +CATALOG = ROOT / "mooninstaller" / "deviceModels.json" MAIN_CPP = ROOT / "src" / "main.cpp" DOCS = ROOT / "docs" diff --git a/moondeck/check/check_esp32_built.py b/moondeck/check/check_esp32_built.py index a6afe87a..74dd801a 100644 --- a/moondeck/check/check_esp32_built.py +++ b/moondeck/check/check_esp32_built.py @@ -32,7 +32,8 @@ SOURCE_DIRS = ("src", "esp32") SOURCE_FILES = ("CMakeLists.txt", "library.json") SOURCE_SUFFIXES = {".c", ".cpp", ".h", ".hpp", ".cmake", ".json", ".txt", ".py", ".js", - ".html", ".css", ".defaults"} + ".html", ".css", ".defaults", ".csv", # .csv: partition tables feed the image + ".yml"} # .yml: idf_component.yml pins components # Build outputs and caches live under the source dirs; they are products, not inputs, and # including them would compare the binary against itself. @@ -149,6 +150,29 @@ def main(): print(f" rebuild: {build_cmd}") return 1 + # MoonBase firmwares carry a second image (built into build/moonbase-/ by the same + # build run): it must exist and be newer than every moonbase/ source, by the same + # sources-not-clock rule as the app image. + import importlib + sys.path.insert(0, str(ROOT / "moondeck" / "build")) + FIRMWARES = importlib.import_module("build_esp32").FIRMWARES + spec = FIRMWARES.get(args.firmware, {}) + if spec.get("moonbase"): + mb_bin = ROOT / "build" / f"moonbase-{spec['chip']}" / "projectMM-moonbase.bin" + if not mb_bin.exists(): + print(f"No MoonBase image for {args.firmware}.") + print(f" expected: {mb_bin.relative_to(ROOT)}") + print(f" build it: {build_cmd}") + return 1 + mb_built = mb_bin.stat().st_mtime + mb_newest = max((f.stat().st_mtime for f in (ROOT / "moonbase").rglob("*") + if f.is_file() and f.suffix in SOURCE_SUFFIXES + and not SKIP_PARTS.intersection(f.parts)), default=0) + if mb_newest > mb_built: + print(f"MoonBase image for {args.firmware} is STALE: a moonbase/ source is newer.") + print(f" rebuild: {build_cmd}") + return 1 + if args.max_age_hours and age_h > args.max_age_hours: print(f"Firmware for {args.firmware} is older than {args.max_age_hours}h " f"({age_h:.1f}h) — no source is newer, but the age rule asks for a rebuild.") diff --git a/moondeck/check/check_firmwares.py b/moondeck/check/check_firmwares.py index c608322b..1dc18e6e 100644 --- a/moondeck/check/check_firmwares.py +++ b/moondeck/check/check_firmwares.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Check that web-installer/firmwares.json is in sync with the FIRMWARES dict. +"""Check that mooninstaller/firmwares.json is in sync with the FIRMWARES dict. firmwares.json is a generated projection of build_esp32.py's FIRMWARES (the single source of truth), read by the CI release matrix, the ESP Web Tools manifest loops, @@ -15,7 +15,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent -COMMITTED = ROOT / "web-installer" / "firmwares.json" +COMMITTED = ROOT / "mooninstaller" / "firmwares.json" # Reuse the generator's projection so the checker and generator can't disagree. sys.path.insert(0, str(ROOT / "moondeck" / "build")) @@ -36,8 +36,8 @@ def main(): # newline) can't cause spurious drift; only the data matters. if actual != expected: print(f"Firmware check: {n} variants, DRIFT") - print(" web-installer/firmwares.json is stale — regenerate with:") - print(" uv run moondeck/build/generate_firmwares.py --out web-installer/firmwares.json") + print(" mooninstaller/firmwares.json is stale — regenerate with:") + print(" uv run moondeck/build/generate_firmwares.py --out mooninstaller/firmwares.json") sys.exit(1) print(f"Firmware check: {n} variants, 0 issue(s)") diff --git a/moondeck/ci/make_ico.py b/moondeck/ci/make_ico.py index 8e0a285c..cf667533 100644 --- a/moondeck/ci/make_ico.py +++ b/moondeck/ci/make_ico.py @@ -9,7 +9,7 @@ Pillow is declared inline (PEP 723) rather than added to the project, because this is the only thing in the tree that needs it and it runs at most once per build. uv fetches it on demand. -The source is the same web-installer/favicon.png the macOS .icns is derived from, so the mark has +The source is the same mooninstaller/favicon.png the macOS .icns is derived from, so the mark has one home. A resize is unavoidable: ICO stores each image's dimensions in a SINGLE byte (with 0 meaning 256), so the 320x320 source cannot be embedded as-is at any size. """ diff --git a/moondeck/ci/package_desktop.py b/moondeck/ci/package_desktop.py index 8a156898..df7b0357 100644 --- a/moondeck/ci/package_desktop.py +++ b/moondeck/ci/package_desktop.py @@ -150,7 +150,7 @@ def package_deb(binary: Path, version: str) -> Path | None: "Categories=Graphics;Utility;\n", encoding="utf-8") icons = stage / "usr" / "share" / "icons" / "hicolor" / "256x256" / "apps" icons.mkdir(parents=True) - fav = ROOT / "web-installer" / "favicon.png" + fav = ROOT / "mooninstaller" / "favicon.png" if fav.exists(): shutil.copy2(fav, icons / "projectmm.png") @@ -246,7 +246,7 @@ def make_icns(dest: Path) -> Path | None: .icns can carry are left out rather than upscaled, since a soft icon reads worse than a smaller crisp one. Swap in a 1024 master and they can be added. """ - src = ROOT / "web-installer" / "favicon.png" + src = ROOT / "mooninstaller" / "favicon.png" if not src.exists() or shutil.which("iconutil") is None: print("package_desktop: no favicon or no iconutil, the app will use the default icon") return None @@ -448,7 +448,7 @@ def windows_icon(version: str) -> Path | None: if built.exists(): return built out = DIST_DIR / "projectMM.ico" - src = ROOT / "web-installer" / "favicon.png" + src = ROOT / "mooninstaller" / "favicon.png" if not src.exists(): print("package_desktop: no favicon to generate an icon from") return None diff --git a/moondeck/docs/mkdocs_hooks.py b/moondeck/docs/mkdocs_hooks.py index 39d02f22..e8b822c5 100644 --- a/moondeck/docs/mkdocs_hooks.py +++ b/moondeck/docs/mkdocs_hooks.py @@ -38,7 +38,7 @@ # ---- source-link rewrite: docs link OUT to repo files the site can't host ---- # The blob base for links that escape docs/ into the repo (src/, moondeck/, test/, -# CLAUDE.md, README.md, web-installer/). The site can't serve these — src/ isn't +# CLAUDE.md, README.md, mooninstaller/). The site can't serve these — src/ isn't # published — so a relative link 404s on the deployed site. Rewrite to an absolute # GitHub blob URL so "source [Foo.h]" resolves everywhere (locally-served preview + # moonmodules.org). A `.h` link for a module that HAS a generated technical page is @@ -53,7 +53,7 @@ _API_MODULES: dict[str, str] = {} # Repo top-level dirs/files a doc may link into but the site doesn't host. -_OUT_OF_DOCS = ("src/", "moondeck/", "test/", "esp32/", "web-installer/", +_OUT_OF_DOCS = ("src/", "moondeck/", "test/", "esp32/", "mooninstaller/", ".github/", "CLAUDE.md", "README.md", "library.json", "CMakeLists.txt", ".clang-tidy", ".clangd") @@ -484,7 +484,7 @@ def on_post_build(config): release.yml does its OWN, more complete install/ staging (incl. the releases// firmware binaries), so this must NOT run there and risk overlaying the binary-less repo copy over it. Mirrors release.yml's - `cp -r web-installer/. pages/install/` + the install-picker*.js + library.json + `cp -r mooninstaller/. pages/install/` + the install-picker*.js + library.json siblings. Best-effort: absent files are skipped.""" import os import shutil @@ -494,7 +494,7 @@ def on_post_build(config): site = Path(config["site_dir"]) dst = site / "install" - src = ROOT / "web-installer" + src = ROOT / "mooninstaller" if not src.is_dir(): return dst.mkdir(parents=True, exist_ok=True) diff --git a/moondeck/event/precommit.py b/moondeck/event/precommit.py index a314c3c6..c0a7a59e 100644 --- a/moondeck/event/precommit.py +++ b/moondeck/event/precommit.py @@ -46,13 +46,13 @@ def build_gates(firmware, full_esp32=False): Gate("device-model catalog", UV + ["moondeck/check/check_devices.py"], - lambda f: touches(f, "web-installer/deviceModels.json", + lambda f: touches(f, "mooninstaller/deviceModels.json", "moondeck/check/check_devices.py")), Gate("firmware list", UV + ["moondeck/check/check_firmwares.py"], lambda f: touches(f, "moondeck/build/build_esp32.py", - "web-installer/firmwares.json", + "mooninstaller/firmwares.json", "moondeck/check/check_firmwares.py")), # The cross-language contracts ctest cannot reach: the Improv frame wire format @@ -64,7 +64,7 @@ def build_gates(firmware, full_esp32=False): Gate("host tests (JS)", ["node", "--test", "test/js/**/*.test.mjs"], - lambda f: touches(f, "web-installer/", "test/js/", "src/ui/")), + lambda f: touches(f, "mooninstaller/", "test/js/", "src/ui/")), # Needs a board plugged in, so it is recommended rather than blocking. Its trigger # is the provisioning path it covers; run_gates drops it from the report entirely @@ -72,7 +72,7 @@ def build_gates(firmware, full_esp32=False): Gate("Improv smoke test", None, lambda f: touches(f, "src/core/ImprovFrame.h", "src/platform/esp32/platform_esp32_improv.cpp", - "web-installer/index.html", "src/ui/install-picker.js", + "mooninstaller/index.html", "src/ui/install-picker.js", "moondeck/build/improv_"), manual_hint="recommended with an ESP32 connected: " "uv run moondeck/build/improv_smoke_test.py --port "), diff --git a/moondeck/moondeck.py b/moondeck/moondeck.py index 9df8638d..b261e459 100644 --- a/moondeck/moondeck.py +++ b/moondeck/moondeck.py @@ -64,11 +64,11 @@ def _app_version(): # Device-model catalog (single source of truth, shared with the web installer) # --------------------------------------------------------------------------- -DEVICE_MODELS_FILE = ROOT / "web-installer" / "deviceModels.json" +DEVICE_MODELS_FILE = ROOT / "mooninstaller" / "deviceModels.json" def _load_device_models(): - """Load web-installer/deviceModels.json. Returns [] on missing/malformed file — + """Load mooninstaller/deviceModels.json. Returns [] on missing/malformed file — `_deduce_device_model` then always returns "" (no firmware uniquely identifies a board), MoonDeck JS shows only the empty default. The web installer Step 2 picker will share this file. @@ -81,11 +81,11 @@ def _load_device_models(): DEVICE_MODELS = _load_device_models() -FIRMWARES_FILE = ROOT / "web-installer" / "firmwares.json" +FIRMWARES_FILE = ROOT / "mooninstaller" / "firmwares.json" def _load_firmwares(): - """Shipping firmware-variant names from web-installer/firmwares.json — the + """Shipping firmware-variant names from mooninstaller/firmwares.json — the generated projection of build_esp32's FIRMWARES dict (the single source of truth, shared with the CI release matrix). Returns [] on missing/malformed file, so the MoonDeck UI just shows no firmware entries. Filtering on @@ -275,7 +275,7 @@ def _deduce_device_model(firmware: str) -> str: """Firmware → deviceModel name when exactly one catalog entry claims this firmware. Returns "" when zero (unknown firmware) or multiple device models claim it (ambiguous — user picks). Catalog lives at - web-installer/deviceModels.json; see docs/architecture.md § Firmware vs board. + mooninstaller/deviceModels.json; see docs/architecture.md § Firmware vs board. """ if not firmware: return "" @@ -286,7 +286,7 @@ def _deduce_device_model(firmware: str) -> str: def _push_device(ip: str, model: str) -> bool: """POST /api/control on the device for every per-board control in deviceModels.json. - For device models that have a catalog entry in web-installer/deviceModels.json: fans + For device models that have a catalog entry in mooninstaller/deviceModels.json: fans out the full `controls..` block (matching the web installer's and the device-side `?deviceModel=` Inject path — same generic iteration, so adding a new field to a deviceModel entry Just Works without @@ -1390,7 +1390,7 @@ def do_GET(self): self._send_json({"modules": test_meta.list_test_modules()}) elif self.path == "/api/device-models": - # Serves web-installer/deviceModels.json (loaded at startup). The web + # Serves mooninstaller/deviceModels.json (loaded at startup). The web # installer (Step 2) fetches the same file directly from Pages; # MoonDeck reads it locally and exposes it here so the JS UI shares # one source of truth with the Python deduce path. diff --git a/moondeck/moondeck_ui/app.js b/moondeck/moondeck_ui/app.js index 922cff2e..4b2e70d8 100644 --- a/moondeck/moondeck_ui/app.js +++ b/moondeck/moondeck_ui/app.js @@ -13,7 +13,7 @@ let firmwares = []; let scenarios = []; // [{name, module, also}] let testModules = []; // ["CamelCaseName", ...] // Device-model catalog loaded from /api/device-models (served by moondeck.py from -// web-installer/deviceModels.json) — the same file the web installer fetches. Empty until +// mooninstaller/deviceModels.json) — the same file the web installer fetches. Empty until // init() loads it; renderDevices waits on init. let deviceModels = []; // [{ name, firmwares: [...], ... }] — `name` is the identifier + label // (single-name catalog, matched by b.name); firmwares[0] is the default. diff --git a/moondeck/qemu/run_qemu.py b/moondeck/qemu/run_qemu.py index d764c35a..fd822e68 100644 --- a/moondeck/qemu/run_qemu.py +++ b/moondeck/qemu/run_qemu.py @@ -56,7 +56,9 @@ def merged_flash(force: bool) -> str: # which made this cache never expire: after a rebuild the emulator kept booting the PREVIOUS # app, and code that was plainly in the .bin appeared not to run at all. app = os.path.join(BUILD, "projectMM.bin") - newest_input = max((os.path.getmtime(p) for p in (args, app) if os.path.exists(p)), default=0) + moonbase = os.path.join(os.path.dirname(BUILD), "moonbase-esp32", "projectMM-moonbase.bin") + newest_input = max((os.path.getmtime(p) for p in (args, app, moonbase) + if os.path.exists(p)), default=0) if os.path.exists(out) and not force and os.path.getmtime(out) > newest_input: return out # find_idf_python returns the venv DIRECTORY; the interpreter is bin/python inside it. @@ -64,8 +66,20 @@ def merged_flash(force: bool) -> str: idf_py = os.path.join(str(venv), "bin", "python") if venv else "" if not idf_py or not os.path.exists(idf_py): sys.exit("no ESP-IDF Python env found, source export.sh, or install the IDF tools") + # The qemu firmware carries MoonBase, and IDF's own flash_args stages the app at the factory + # offset (MoonBase's slot): the same correction every flasher applies. moonbase_flash_files + # is the one place that knows the corrected layout; the flat list it returns feeds merge_bin + # directly instead of @flash_args. + sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "build")) + from build_esp32 import FIRMWARES, moonbase_flash_files + if FIRMWARES["qemu"].get("moonbase"): + from pathlib import Path + writes = [str(x) for off, path in moonbase_flash_files("qemu", Path(BUILD)) + for x in (off, path)] + else: + writes = [f"@{args}"] r = subprocess.run([idf_py, "-m", "esptool", "--chip", "esp32", "merge_bin", - "-o", out, "--fill-flash-size", "4MB", f"@{args}"], + "-o", out, "--fill-flash-size", "4MB"] + writes, cwd=BUILD, capture_output=True, text=True) if r.returncode != 0: sys.exit(f"merge_bin failed:\n{r.stderr[:800]}") diff --git a/moondeck/run/preview_installer.py b/moondeck/run/preview_installer.py index 74ef09e5..df17ed94 100644 --- a/moondeck/run/preview_installer.py +++ b/moondeck/run/preview_installer.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Locally preview the web installer at web-installer/index.html. +"""Locally preview the web installer at mooninstaller/index.html. Stages a small directory (the install page + the shared install-picker module) and serves it with `python -m http.server`, plus the @@ -12,7 +12,7 @@ - **render-only** (no `build/esp32-*/projectMM.bin` present): the picker populates against the real GitHub Releases API, dropdowns work, but clicking Install fails because the local server has no - `releases/` tree. Equivalent to "Recipe A" in web-installer/README.md. + `releases/` tree. Equivalent to "Recipe A" in mooninstaller/README.md. Useful for iterating on HTML/CSS/JS without tagging a release. - **flash-ready** (at least one local ESP32 build exists): the @@ -43,10 +43,15 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent.parent -INSTALL_DIR = ROOT / "web-installer" + +# build_esp32.py is the single source of truth for firmware variants (same import +# generate_manifest.py and collect_kpi.py use). +sys.path.insert(0, str(ROOT / "moondeck" / "build")) +from build_esp32 import FIRMWARES # noqa: E402 +INSTALL_DIR = ROOT / "mooninstaller" ASSETS_BOARDS_DIR = ROOT / "docs" / "assets" / "boards" PICKER_JS = ROOT / "src" / "ui" / "install-picker.js" -# Board-catalog / chip-detection half of the picker — web-installer only (not +# Board-catalog / chip-detection half of the picker — mooninstaller only (not # embedded in firmware), imported by index.html. Staged alongside PICKER_JS. PICKER_BOARDS_JS = ROOT / "src" / "ui" / "install-picker-boards.js" STAGE_DIR = ROOT / "build" / "install-preview" @@ -77,9 +82,9 @@ def _stage_runtime_files(src_dir: Path, dst_dir: Path): """Copy every browser-loadable file (.html/.js/.css/.json/.png/.ico/.svg) from - src_dir to dst_dir — mirrors release.yml's `cp -r web-installer/. pages/install/`. + src_dir to dst_dir — mirrors release.yml's `cp -r mooninstaller/. pages/install/`. README.md / other .md are docs, skipped. The deploy's `cp -r` is recursive, so a - subdirectory of static assets (web-installer/assets/, the app-store badges) is + subdirectory of static assets (mooninstaller/assets/, the app-store badges) is staged too — walk the tree rather than just the top level, or those 404 in preview while working in production.""" exts = (".html", ".js", ".css", ".json", ".png", ".ico", ".svg") @@ -207,6 +212,14 @@ def stage_local_builds(builds: list[Path]) -> list[str]: releases_dir / f"partition-table-{size}.bin") shutil.copy(build_dir / "ota_data_initial.bin", releases_dir / "shared-ota-data.bin") + # MoonBase firmwares also ship the shared maintenance image + the slot-0 otadata + # their manifests reference (same names release.yml stages). + if FIRMWARES.get(firmware, {}).get("moonbase"): + from build_esp32 import otadata_slot0_bytes + chip = FIRMWARES[firmware]["chip"] + shutil.copy(build_dir.parent / f"moonbase-{chip}" / "projectMM-moonbase.bin", + releases_dir / f"shared-moonbase-{chip}.bin") + (releases_dir / "shared-ota-data-slot0.bin").write_bytes(otadata_slot0_bytes()) except FileNotFoundError as e: # Partial build (bootloader / partition-table missing) — skip this # firmware rather than half-stage it, the picker would offer it diff --git a/web-installer/README.md b/mooninstaller/README.md similarity index 99% rename from web-installer/README.md rename to mooninstaller/README.md index 7c94963f..0448c35f 100644 --- a/web-installer/README.md +++ b/mooninstaller/README.md @@ -319,7 +319,7 @@ for F in esp32 esp32-eth esp32s3-n16r8; do done # Drop the install page + shared picker module in place. -cp web-installer/index.html "$DIST"/ +cp mooninstaller/index.html "$DIST"/ cp src/ui/install-picker.js "$DIST"/ cd "$DIST" && python3 -m http.server 8000 @@ -376,6 +376,6 @@ don't ship the API. Manual setup, one-time per repo: **Settings → Pages → Source: GitHub Actions**. No deploy-from-branch — the workflow is the only producer. A separate -`web-installer/`-only Pages deploy was considered and rejected: it would +`mooninstaller/`-only Pages deploy was considered and rejected: it would have to re-run the same cumulative-content dance, so a docs-only deploy buys nothing. diff --git a/web-installer/assets/app-store-badge.svg b/mooninstaller/assets/app-store-badge.svg similarity index 100% rename from web-installer/assets/app-store-badge.svg rename to mooninstaller/assets/app-store-badge.svg diff --git a/web-installer/assets/google-play-badge.png b/mooninstaller/assets/google-play-badge.png similarity index 100% rename from web-installer/assets/google-play-badge.png rename to mooninstaller/assets/google-play-badge.png diff --git a/web-installer/assets/home-assistant-icon.png b/mooninstaller/assets/home-assistant-icon.png similarity index 100% rename from web-installer/assets/home-assistant-icon.png rename to mooninstaller/assets/home-assistant-icon.png diff --git a/web-installer/config-ops.js b/mooninstaller/config-ops.js similarity index 100% rename from web-installer/config-ops.js rename to mooninstaller/config-ops.js diff --git a/web-installer/deviceModels.json b/mooninstaller/deviceModels.json similarity index 99% rename from web-installer/deviceModels.json rename to mooninstaller/deviceModels.json index c50cedec..8d8df58d 100644 --- a/web-installer/deviceModels.json +++ b/mooninstaller/deviceModels.json @@ -2,6 +2,7 @@ { "name": "Olimex ESP32-Gateway Rev G", "chip": "ESP32", + "flashBaud": 460800, "firmwares": [ "esp32", "esp32-eth" diff --git a/web-installer/devices.js b/mooninstaller/devices.js similarity index 100% rename from web-installer/devices.js rename to mooninstaller/devices.js diff --git a/web-installer/favicon.png b/mooninstaller/favicon.png similarity index 100% rename from web-installer/favicon.png rename to mooninstaller/favicon.png diff --git a/web-installer/firmwares.json b/mooninstaller/firmwares.json similarity index 100% rename from web-installer/firmwares.json rename to mooninstaller/firmwares.json diff --git a/web-installer/improv-frame.js b/mooninstaller/improv-frame.js similarity index 100% rename from web-installer/improv-frame.js rename to mooninstaller/improv-frame.js diff --git a/web-installer/index.html b/mooninstaller/index.html similarity index 99% rename from web-installer/index.html rename to mooninstaller/index.html index 9d3ffcd4..8fb763bf 100644 --- a/web-installer/index.html +++ b/mooninstaller/index.html @@ -64,7 +64,7 @@

projectMM Installer