diff --git a/examples/python/ess-hello-jwt-auth-code/.env.example b/examples/python/ess-hello-jwt-auth-code/.env.example new file mode 100644 index 0000000..bb39e6b --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/.env.example @@ -0,0 +1,23 @@ +# OpenID Connect / OAuth 2.0 Authorization Code Flow with PKCE +# +# Register a "web" application with your OIDC provider (Okta, Auth0, +# Microsoft Entra, Google, Keycloak, etc.) configured for: +# Grant type: Authorization Code (with PKCE) +# Redirect URI: http://localhost:8900/auth/callback +# +# The two URLs below come from your provider's OpenID Connect metadata, +# typically discoverable at: /.well-known/openid-configuration + +# Required: authorization endpoint (where the user logs in) +OIDC_AUTHORIZE_URL= + +# Required: token endpoint (where the auth code is exchanged for tokens) +OIDC_TOKEN_URL= + +# Required: client credentials issued by your provider +OIDC_CLIENT_ID= +OIDC_CLIENT_SECRET= + +# Optional overrides +# OIDC_SCOPE=openid +# OIDC_REDIRECT_URI=http://localhost:8900/auth/callback diff --git a/examples/python/ess-hello-jwt-auth-code/README.md b/examples/python/ess-hello-jwt-auth-code/README.md new file mode 100644 index 0000000..02dc2b0 --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/README.md @@ -0,0 +1,96 @@ +# ess-hello-jwt-auth-code + +Obtain a JWT access token from any [OpenID Connect](https://openid.net/developers/how-connect-works/) +provider via the [OAuth 2.0 Authorization Code Flow](https://datatracker.ietf.org/doc/html/rfc6749#section-4.1) +with [PKCE](https://datatracker.ietf.org/doc/html/rfc7636). + +Run the script, log in through your browser, and copy the printed token. + +```bash +uv run ess-hello-jwt-auth-code +# Opens browser -> provider login -> prints token to stdout +``` + +Expected output: + +``` +Opening browser for OIDC login... +Waiting for callback on localhost:8900 ... +Exchanging code for token... + +--- ACCESS TOKEN (copy below this line) --- +eyJhbGciOiJSUzI1NiIs... +--- END TOKEN --- + +Expires in: 3600s +Token type: Bearer +``` + +The token is printed to **stdout** (status messages go to stderr), so you +can pipe it directly: + +```bash +uv run ess-hello-jwt-auth-code > token.txt +``` + +## Installation + +From the workspace root: + +```bash +uv sync --all-packages +``` + +## Configuration + +From the example directory: + +```bash +cd examples/python/ess-hello-jwt-auth-code +``` + +Copy `.env.example` to `.env` and fill in your credentials: + +```bash +cp .env.example .env +``` + +Register a **web** app with your OpenID Connect provider configured for: + +- **Grant type**: Authorization Code (with PKCE) +- **Redirect URI**: `http://localhost:8900/auth/callback` + +The `OIDC_AUTHORIZE_URL` and `OIDC_TOKEN_URL` values come from your +provider's OpenID Connect discovery document, typically published at +`/.well-known/openid-configuration`. Provider documentation: + +- Okta: +- Auth0: +- Microsoft Entra: +- Google: +- Keycloak: + +| Variable | Required | Description | Default | +| -------------------- | -------- | ---------------------------------------------------------- | ---------------------------------------- | +| `OIDC_AUTHORIZE_URL` | Yes | Provider's authorization endpoint (where the user logs in) | -- | +| `OIDC_TOKEN_URL` | Yes | Provider's token endpoint (where the code is exchanged) | -- | +| `OIDC_CLIENT_ID` | Yes | OAuth 2.0 client ID issued by the provider | -- | +| `OIDC_CLIENT_SECRET` | Yes | OAuth 2.0 client secret issued by the provider | -- | +| `OIDC_SCOPE` | No | Scopes to request (space-separated) | `openid` | +| `OIDC_REDIRECT_URI` | No | Local callback URL (port is parsed from this value) | `http://localhost:8900/auth/callback` | + +## How it works + +1. Generates a fresh PKCE `code_verifier` / `code_challenge` and a CSRF + `state` value. +2. Opens the provider's authorization URL in your default browser. +3. Spins up a one-shot HTTP listener on `localhost` to capture the + redirect callback. +4. Validates the returned `state` matches what was sent. +5. Exchanges the authorization code (plus the `code_verifier`) at the + token endpoint for an access token. +6. Prints the access token to stdout; status messages go to stderr. + +## License + +[Apache 2.0](https://choosealicense.com/licenses/apache-2.0/) diff --git a/examples/python/ess-hello-jwt-auth-code/pyproject.toml b/examples/python/ess-hello-jwt-auth-code/pyproject.toml new file mode 100644 index 0000000..3b13ad4 --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "ess-hello-jwt-auth-code" +version = "0.1.0" +description = "Obtain a JWT access token from any OpenID Connect provider via the OAuth 2.0 Authorization Code Flow with PKCE" +readme = "README.md" +requires-python = ">=3.12,<3.13" +license = { text = "Apache-2.0" } +keywords = ["oauth2", "oidc", "jwt", "pkce", "authorization-code"] +dependencies = [ + "httpx>=0.28.0", + "python-dotenv>=1.1.0", +] + +[project.scripts] +ess-hello-jwt-auth-code = "ess_hello_jwt_auth_code.main:main" + +[dependency-groups] +dev = ["ruff>=0.14.2", "pytest>=8.0.0"] + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/ess_hello_jwt_auth_code"] +exclude = ["**/test_*.py"] diff --git a/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/__init__.py b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/__init__.py new file mode 100644 index 0000000..9881313 --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/__init__.py @@ -0,0 +1 @@ +# SPDX-License-Identifier: Apache-2.0 diff --git a/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/main.py b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/main.py new file mode 100644 index 0000000..969d977 --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/main.py @@ -0,0 +1,271 @@ +# SPDX-License-Identifier: Apache-2.0 +"""OAuth 2.0 Authorization Code Flow with PKCE against any OIDC provider. + +Opens a browser for the user to authenticate, captures the callback on a +temporary local HTTP server, exchanges the code for tokens, and prints the +access token (JWT) to stdout for copy-paste. +""" + +from __future__ import annotations + +import base64 +import dataclasses +import hashlib +import os +import secrets +import sys +import time +import webbrowser +from http import HTTPStatus +from http.client import HTTP_PORT +from http.server import BaseHTTPRequestHandler, HTTPServer +from urllib.parse import parse_qs, urlencode, urlparse + +import httpx +from dotenv import load_dotenv + +_DEFAULT_REDIRECT_URI = "http://localhost:8900/auth/callback" +_DEFAULT_SCOPE = "openid" + +_REQUEST_TIMEOUT = 30 +_CALLBACK_TIMEOUT = 120 + + +class _AuthCodeError(Exception): + """Raised when the authorization code flow fails.""" + + +@dataclasses.dataclass(frozen=True) +class _OidcClient: + """OIDC provider endpoints and client credentials.""" + + authorize_url: str + token_url: str + client_id: str + client_secret: str + redirect_uri: str + scope: str + + +def _generate_pkce() -> tuple[str, str]: + """Return ``(code_verifier, code_challenge)`` for PKCE (S256).""" + verifier = secrets.token_urlsafe(64) + digest = hashlib.sha256(verifier.encode("ascii")).digest() + challenge = base64.urlsafe_b64encode(digest).rstrip(b"=").decode("ascii") + return verifier, challenge + + +def _make_callback_handler( + expected_state: str, + result: dict[str, str | None], +) -> type[BaseHTTPRequestHandler]: + """Return a request handler class that captures the OAuth callback.""" + + class _CallbackHandler(BaseHTTPRequestHandler): + def do_GET(self) -> None: # noqa: N802 + query = parse_qs(urlparse(self.path).query) + + returned_state = query.get("state", [None])[0] + if returned_state != expected_state: + self._respond( + HTTPStatus.BAD_REQUEST, + "OAuth state mismatch -- possible CSRF attack", + ) + return + + if "error" in query: + result["error"] = ( + f"{query['error'][0]}: " + f"{query.get('error_description', ['unknown'])[0]}" + ) + self._respond(HTTPStatus.BAD_REQUEST, result["error"]) + return + + result["code"] = query.get("code", [None])[0] + if not result["code"]: + result["error"] = "No authorization code in callback" + self._respond(HTTPStatus.BAD_REQUEST, result["error"]) + return + + self._respond( + HTTPStatus.OK, + "Authenticated! You can close this tab.", + ) + + def _respond(self, status: HTTPStatus, body: str) -> None: + self.send_response(status) + self.send_header("Content-Type", "text/plain; charset=utf-8") + self.end_headers() + self.wfile.write(body.encode()) + + def log_message(self, format: str, *args: object) -> None: # noqa: A002 + pass + + return _CallbackHandler + + +def _bind_callback_server(port: int, expected_state: str) -> tuple[HTTPServer, dict]: + """Bind the callback server so it is ready before the browser opens.""" + result: dict[str, str | None] = {"code": None, "error": None} + handler_cls = _make_callback_handler(expected_state, result) + try: + server = HTTPServer(("localhost", port), handler_cls) + except OSError as exc: + raise _AuthCodeError(f"Could not bind to localhost:{port}: {exc}") from exc + return server, result + + +def _capture_callback(port: int, expected_state: str) -> str: + """Bind a server and wait for the OAuth callback.""" + server, result = _bind_callback_server(port, expected_state) + return _wait_for_callback(server, result) + + +def _wait_for_callback(server: HTTPServer, result: dict) -> str: + """Wait for the OAuth callback on an already-bound server.""" + deadline = time.monotonic() + _CALLBACK_TIMEOUT + try: + while result["code"] is None and result["error"] is None: + remaining = deadline - time.monotonic() + if remaining <= 0: + raise _AuthCodeError( + f"No callback received within {_CALLBACK_TIMEOUT}s" + ) + server.timeout = remaining + server.handle_request() + finally: + server.server_close() + + if result["error"]: + raise _AuthCodeError(result["error"]) + if not result["code"]: + raise _AuthCodeError("No authorization code received") + return result["code"] + + +def _exchange_code( + client: _OidcClient, + code: str, + code_verifier: str, +) -> dict: + """Exchange an authorization code for tokens.""" + with httpx.Client(timeout=_REQUEST_TIMEOUT) as http: + response = http.post( + client.token_url, + data={ + "grant_type": "authorization_code", + "client_id": client.client_id, + "client_secret": client.client_secret, + "code": code, + "redirect_uri": client.redirect_uri, + "code_verifier": code_verifier, + }, + ) + if response.status_code != HTTPStatus.OK: + raise _AuthCodeError(f"Token exchange failed: {response.text}") + return response.json() + + +def _require_env(name: str) -> str: + value = os.environ.get(name, "") + if not value: + print( # noqa: T201 + f"Error: set {name} in .env (see .env.example)", + file=sys.stderr, + ) + sys.exit(1) + return value + + +def main() -> None: + """Run the authorization code flow and print the resulting token.""" + load_dotenv() + + client = _OidcClient( + authorize_url=_require_env("OIDC_AUTHORIZE_URL"), + token_url=_require_env("OIDC_TOKEN_URL"), + client_id=_require_env("OIDC_CLIENT_ID"), + client_secret=_require_env("OIDC_CLIENT_SECRET"), + redirect_uri=os.environ.get("OIDC_REDIRECT_URI", _DEFAULT_REDIRECT_URI), + scope=os.environ.get("OIDC_SCOPE", _DEFAULT_SCOPE), + ) + parsed_uri = urlparse(client.redirect_uri) + if parsed_uri.scheme != "http": + print( # noqa: T201 + f"Error: only http:// redirect URIs are supported" + f" (got {parsed_uri.scheme}://)", + file=sys.stderr, + ) + sys.exit(1) + port = parsed_uri.port or HTTP_PORT + + state = secrets.token_urlsafe(32) + code_verifier, code_challenge = _generate_pkce() + + params = urlencode( + { + "client_id": client.client_id, + "response_type": "code", + "redirect_uri": client.redirect_uri, + "scope": client.scope, + "state": state, + "code_challenge": code_challenge, + "code_challenge_method": "S256", + } + ) + auth_url = f"{client.authorize_url}?{params}" + + try: + server, result = _bind_callback_server(port, expected_state=state) + except _AuthCodeError as exc: + print(f"Error: {exc}", file=sys.stderr) # noqa: T201 + sys.exit(1) + + print("Opening browser for OIDC login...", file=sys.stderr) # noqa: T201 + if not webbrowser.open(auth_url): + print( # noqa: T201 + f"Could not open browser. Visit this URL manually:\n{auth_url}", + file=sys.stderr, + ) + + print( # noqa: T201 + f"Waiting for callback on localhost:{port} ...", + file=sys.stderr, + ) + + try: + code = _wait_for_callback(server, result) + except _AuthCodeError as exc: + print(f"Error: {exc}", file=sys.stderr) # noqa: T201 + sys.exit(1) + + print("Exchanging code for token...", file=sys.stderr) # noqa: T201 + + try: + token_data = _exchange_code(client, code, code_verifier) + except _AuthCodeError as exc: + print(f"Error: {exc}", file=sys.stderr) # noqa: T201 + sys.exit(1) + except httpx.RequestError as exc: + print( # noqa: T201 + f"Error: network failure during token exchange: {exc}", + file=sys.stderr, + ) + sys.exit(1) + + access_token = token_data["access_token"] + expires_in = token_data.get("expires_in", "unknown") + + print("", file=sys.stderr) # noqa: T201 + print("--- ACCESS TOKEN (copy below this line) ---", file=sys.stderr) # noqa: T201 + print(access_token) # noqa: T201 + print("--- END TOKEN ---", file=sys.stderr) # noqa: T201 + print(f"\nExpires in: {expires_in}s", file=sys.stderr) # noqa: T201 + print( # noqa: T201 + f"Token type: {token_data.get('token_type', 'unknown')}", + file=sys.stderr, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/test_main.py b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/test_main.py new file mode 100644 index 0000000..03b67e4 --- /dev/null +++ b/examples/python/ess-hello-jwt-auth-code/src/ess_hello_jwt_auth_code/test_main.py @@ -0,0 +1,218 @@ +# ruff: noqa: S106, PLR2004 -- test credentials are not real secrets; magic numbers expected +"""Tests for the OAuth 2.0 Authorization Code Flow.""" + +from __future__ import annotations + +import threading +import time +from http import HTTPStatus +from unittest.mock import MagicMock, patch +from urllib.parse import urlencode + +import httpx +import pytest + +from ess_hello_jwt_auth_code.main import ( + _AuthCodeError, + _capture_callback, + _exchange_code, + _generate_pkce, + _OidcClient, +) + +_TEST_STATE = "test-state-abc123" + + +def _make_client(**overrides: str) -> _OidcClient: + defaults = { + "authorize_url": "https://idp.example.com/authorize", + "token_url": "https://idp.example.com/token", + "client_id": "test-client-id", + "client_secret": "test-client-secret", + "redirect_uri": "http://localhost:8900/auth/callback", + "scope": "openid", + } + defaults.update(overrides) + return _OidcClient(**defaults) + + +class TestGeneratePkce: + def test_returns_verifier_and_challenge(self): + verifier, challenge = _generate_pkce() + assert len(verifier) > 40 + assert len(challenge) > 20 + + def test_verifier_is_url_safe(self): + verifier, _ = _generate_pkce() + assert all(c.isalnum() or c in "-_" for c in verifier) + + def test_challenge_is_url_safe_base64(self): + _, challenge = _generate_pkce() + assert "=" not in challenge + assert all(c.isalnum() or c in "-_" for c in challenge) + + def test_generates_unique_values(self): + pairs = [_generate_pkce() for _ in range(5)] + verifiers = [v for v, _ in pairs] + assert len(set(verifiers)) == 5 + + +class TestCaptureCallback: + _RETRY_INTERVAL = 0.05 + _RETRY_TIMEOUT = 5.0 + + def _send_callback( + self, + port: int, + query_params: dict, + *, + wait_for: threading.Event | None = None, + done_event: threading.Event | None = None, + ): + """Send a GET to the callback server, retrying until it connects.""" + url = f"http://localhost:{port}/auth/callback?{urlencode(query_params)}" + + def _request(): + if wait_for: + wait_for.wait(timeout=self._RETRY_TIMEOUT) + deadline = time.monotonic() + self._RETRY_TIMEOUT + while time.monotonic() < deadline: + try: + httpx.get(url, timeout=2) + if done_event: + done_event.set() + return + except httpx.ConnectError: + time.sleep(self._RETRY_INTERVAL) + except httpx.RequestError: + if done_event: + done_event.set() + pass + + thread = threading.Thread(target=_request, daemon=True) + thread.start() + return thread + + def test_success(self): + port = 18901 + self._send_callback(port, {"state": _TEST_STATE, "code": "auth-code-xyz"}) + result = _capture_callback(port, expected_state=_TEST_STATE) + assert result == "auth-code-xyz" + + def test_state_mismatch_keeps_serving(self): + """Wrong-state requests keep the server alive for the next valid one.""" + port = 18902 + bad_done = threading.Event() + self._send_callback( + port, + {"state": "wrong-state", "code": "bad"}, + done_event=bad_done, + ) + self._send_callback( + port, + {"state": _TEST_STATE, "code": "good-code"}, + wait_for=bad_done, + ) + result = _capture_callback(port, expected_state=_TEST_STATE) + assert result == "good-code" + + def test_idp_error_raises(self): + port = 18903 + self._send_callback( + port, + { + "state": _TEST_STATE, + "error": "access_denied", + "error_description": "User cancelled", + }, + ) + with pytest.raises(_AuthCodeError, match="access_denied: User cancelled"): + _capture_callback(port, expected_state=_TEST_STATE) + + def test_timeout_raises(self): + port = 18904 + with ( + patch("ess_hello_jwt_auth_code.main._CALLBACK_TIMEOUT", 0.1), + pytest.raises(_AuthCodeError, match="No callback received"), + ): + _capture_callback(port, expected_state=_TEST_STATE) + + def test_response_is_plain_text(self): + """Verify the response uses text/plain, preventing XSS.""" + port = 18905 + response_holder: list[httpx.Response] = [] + url = ( + f"http://localhost:{port}/auth/callback?" + f"{urlencode({'state': _TEST_STATE, 'code': 'c'})}" + ) + + def _request(): + deadline = time.monotonic() + self._RETRY_TIMEOUT + while time.monotonic() < deadline: + try: + resp = httpx.get(url, timeout=2) + response_holder.append(resp) + return + except httpx.ConnectError: + time.sleep(self._RETRY_INTERVAL) + except httpx.RequestError: + pass + + thread = threading.Thread(target=_request, daemon=True) + thread.start() + _capture_callback(port, expected_state=_TEST_STATE) + thread.join(timeout=2) + + assert response_holder, "No response captured" + content_type = response_holder[0].headers.get("content-type", "") + assert "text/plain" in content_type + assert "

" not in response_holder[0].text + + +class TestExchangeCode: + @staticmethod + def _setup_mock_client(mock_client_cls, mock_response): + mock_post = MagicMock(return_value=mock_response) + mock_http = MagicMock(post=mock_post) + mock_client_cls.return_value.__enter__ = MagicMock(return_value=mock_http) + mock_client_cls.return_value.__exit__ = MagicMock(return_value=False) + return mock_post + + def test_success(self): + mock_response = MagicMock() + mock_response.status_code = HTTPStatus.OK + mock_response.json.return_value = { + "access_token": "jwt-token", + "token_type": "Bearer", + "expires_in": 3600, + } + + with patch("ess_hello_jwt_auth_code.main.httpx.Client") as mock_client_cls: + mock_post = self._setup_mock_client(mock_client_cls, mock_response) + result = _exchange_code(_make_client(), "auth-code", "test-verifier") + + assert result["access_token"] == "jwt-token" + assert result["expires_in"] == 3600 + + mock_post.assert_called_once() + call_data = mock_post.call_args.kwargs["data"] + assert call_data["grant_type"] == "authorization_code" + assert call_data["client_id"] == "test-client-id" + assert call_data["code"] == "auth-code" + assert call_data["code_verifier"] == "test-verifier" + + def test_failure_raises(self): + mock_response = MagicMock() + mock_response.status_code = HTTPStatus.BAD_REQUEST + mock_response.text = "invalid_grant" + + with patch("ess_hello_jwt_auth_code.main.httpx.Client") as mock_client_cls: + mock_post = self._setup_mock_client(mock_client_cls, mock_response) + + with pytest.raises(_AuthCodeError, match="Token exchange failed"): + _exchange_code(_make_client(), "bad-code", "verifier") + + mock_post.assert_called_once() + call_data = mock_post.call_args.kwargs["data"] + assert call_data["grant_type"] == "authorization_code" + assert call_data["code"] == "bad-code" diff --git a/pyproject.toml b/pyproject.toml index e4a0194..fe5e7e5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ members = [ "packages/python/langsmith-network", "packages/python/azure-ai", "examples/python/ess-messages", + "examples/python/ess-hello-jwt-auth-code", ] [tool.ruff] diff --git a/tools/typescript/essentials-sync/README.md b/tools/typescript/essentials-sync/README.md index a776938..270bc5f 100644 --- a/tools/typescript/essentials-sync/README.md +++ b/tools/typescript/essentials-sync/README.md @@ -118,6 +118,8 @@ The `opus` sentinel resolves to the highest-version Opus model on your Cursor ac In extract mode (the default), scanners only run against the **extracted package**, not the wrapper. That is deliberate: the wrapper exists to hold the company-specific defaults that the open-source library cannot ship with. In `--no-extract` mode, scanners run against the sync target (the essentials copy) as before. +Files whose basename starts with `tmp-` or `tmp.` are skipped by the deterministic scanners and the adversarial reviewer. Use this for working notes the agent leaves for you (the `tmp-RECOMMENDATIONS.md` it writes when the source has heavy jargon, for example) -- target repos are expected to gitignore the `tmp[-.]*` pattern so these files never get committed. + ## Exit codes - `0` -- both phases clean; review and commit yourself. diff --git a/tools/typescript/essentials-sync/package.json b/tools/typescript/essentials-sync/package.json index 68d5ca5..cb39245 100644 --- a/tools/typescript/essentials-sync/package.json +++ b/tools/typescript/essentials-sync/package.json @@ -23,6 +23,7 @@ "chalk": "^5.3.0", "commander": "^12.1.0", "execa": "^9.5.1", + "globby": "^14.1.0", "secretlint": "^9.0.0", "@secretlint/secretlint-rule-preset-recommend": "^9.0.0" }, diff --git a/tools/typescript/essentials-sync/src/prompts.ts b/tools/typescript/essentials-sync/src/prompts.ts index 62e52eb..0cff056 100644 --- a/tools/typescript/essentials-sync/src/prompts.ts +++ b/tools/typescript/essentials-sync/src/prompts.ts @@ -72,11 +72,13 @@ export function buildSyncInitialPrompt({ : "No source pre-scan was performed (--no-source-scan)."; const generalizationNote = jargonHeavy - ? `The source contains substantial company-specific jargon. Also write a file named RECOMMENDATIONS.md inside the target directory that: + ? `The source contains substantial company-specific jargon. Also write a file named tmp-RECOMMENDATIONS.md inside the target directory that: - proposes a new ess-* package name and rationale - lists identifiers and strings that were renamed - - calls out any modules or behaviors that could not be cleanly generalized and should be left behind in the private repo` - : "Source pre-scan looks light on company jargon; no RECOMMENDATIONS.md needed unless you find more during the rewrite."; + - calls out any modules or behaviors that could not be cleanly generalized and should be left behind in the private repo + +The 'tmp-' prefix is required: target repos gitignore 'tmp[-.]*' so this file stays out of commits, and the deterministic scanners and adversarial reviewer skip 'tmp-*' files. Treat it as a working note for the human reviewer; it is allowed to mention internal context (codenames, prior infrastructure, migration history) that would never be acceptable in a public file. Do NOT name the file RECOMMENDATIONS.md (no prefix) -- that name is committed and scanned.` + : "Source pre-scan looks light on company jargon; no tmp-RECOMMENDATIONS.md needed unless you find more during the rewrite."; return `MODE: ${mode} SOURCE (read-only, absolute path): ${plan.sourceAbs} @@ -236,7 +238,9 @@ export function buildReviewerPrompt({ const root = reviewRootAbs ?? plan.targetAbs; return `You are a strict open-source release reviewer. The package at ${root} is about to be published to a public, open-source repository. -Read every file under that path. Flag anything that would embarrass the team or leak company information if released. Be paranoid. Pay particular attention to: +Read every file under that path EXCEPT files whose basename starts with 'tmp-' or 'tmp.' -- those are working notes for the human reviewer, gitignored, and never committed. Skip them entirely; do not flag anything in them. + +For everything else, flag anything that would embarrass the team or leak company information if released. Be paranoid. Pay particular attention to: - Internal hostnames, URLs, account IDs, project codenames. - Personally identifiable information (employee names, IDs, emails, phone numbers). - Subtle references in docstrings, comments, naming, error messages, or example data that imply an internal audience. diff --git a/tools/typescript/essentials-sync/src/scanners/jargon.ts b/tools/typescript/essentials-sync/src/scanners/jargon.ts index 62662d9..68d7add 100644 --- a/tools/typescript/essentials-sync/src/scanners/jargon.ts +++ b/tools/typescript/essentials-sync/src/scanners/jargon.ts @@ -1,18 +1,6 @@ -import { promises as fs } from "node:fs"; -import path from "node:path"; import type { Finding, Severity } from "../types.js"; -import { loadJargonConfig, type JargonConfig } from "../jargon-list.js"; - -const DEFAULT_BINARY_EXTENSIONS = new Set([ - ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf", ".zip", ".tar", ".gz", - ".bz2", ".xz", ".7z", ".woff", ".woff2", ".eot", ".ttf", ".otf", ".ico", - ".so", ".dylib", ".dll", ".class", ".jar", ".wasm", ".bin", -]); - -const SKIP_DIRECTORIES = new Set([ - "node_modules", ".git", ".venv", "venv", "__pycache__", ".pytest_cache", - ".ruff_cache", ".mypy_cache", "dist", "build", ".pulumi", -]); +import { loadJargonConfig } from "../jargon-list.js"; +import { scanTextFilesByLine } from "./text-files.js"; export interface JargonScanOptions { severity: Severity; @@ -24,64 +12,18 @@ export async function scanJargon( options: JargonScanOptions, ): Promise { const config = loadJargonConfig(options.extraTerms); - const findings: Finding[] = []; - for await (const filePath of walkTextFiles(rootPath)) { - const content = await fs.readFile(filePath, "utf8"); - findings.push(...scanContent(filePath, content, config, options.severity)); - } - return findings; -} - -function scanContent( - filePath: string, - content: string, - config: JargonConfig, - severity: Severity, -): Finding[] { - const lines = content.split(/\r?\n/); - const found: Finding[] = []; - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i] ?? ""; - for (const pattern of config.patterns) { - if (pattern.regex.test(line)) { - found.push({ - file: filePath, - line: i + 1, - type: pattern.term.startsWith("*.") ? "internal-url" : "jargon", - severity, - source: "jargon", - message: pattern.message, - term: pattern.term, - }); - } - } - } - return found; -} - -async function* walkTextFiles(rootPath: string): AsyncGenerator { - const stack: string[] = [rootPath]; - while (stack.length > 0) { - const current = stack.pop(); - if (!current) continue; - let entries; - try { - entries = await fs.readdir(current, { withFileTypes: true }); - } catch { - continue; - } - for (const entry of entries) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (SKIP_DIRECTORIES.has(entry.name)) continue; - stack.push(fullPath); - } else if (entry.isFile()) { - const ext = path.extname(entry.name).toLowerCase(); - if (DEFAULT_BINARY_EXTENSIONS.has(ext)) continue; - const stat = await fs.stat(fullPath); - if (stat.size > 5 * 1024 * 1024) continue; - yield fullPath; - } - } - } + return scanTextFilesByLine( + rootPath, + config.patterns, + (pattern, line) => pattern.regex.test(line), + (pattern, filePath, lineNumber) => ({ + file: filePath, + line: lineNumber, + type: pattern.term.startsWith("*.") ? "internal-url" : "jargon", + severity: options.severity, + source: "jargon", + message: pattern.message, + term: pattern.term, + }), + ); } diff --git a/tools/typescript/essentials-sync/src/scanners/pii.ts b/tools/typescript/essentials-sync/src/scanners/pii.ts index 30563e0..d6ee707 100644 --- a/tools/typescript/essentials-sync/src/scanners/pii.ts +++ b/tools/typescript/essentials-sync/src/scanners/pii.ts @@ -1,17 +1,5 @@ -import { promises as fs } from "node:fs"; -import path from "node:path"; import type { Finding, Severity } from "../types.js"; - -const SKIP_DIRECTORIES = new Set([ - "node_modules", ".git", ".venv", "venv", "__pycache__", ".pytest_cache", - ".ruff_cache", ".mypy_cache", "dist", "build", ".pulumi", -]); - -const BINARY_EXTENSIONS = new Set([ - ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf", ".zip", ".tar", ".gz", - ".bz2", ".xz", ".7z", ".woff", ".woff2", ".eot", ".ttf", ".otf", ".ico", - ".so", ".dylib", ".dll", ".class", ".jar", ".wasm", ".bin", -]); +import { scanTextFilesByLine } from "./text-files.js"; interface PiiPattern { name: string; @@ -51,60 +39,22 @@ export async function scanPii( rootPath: string, options: PiiScanOptions, ): Promise { - const findings: Finding[] = []; - for await (const filePath of walkTextFiles(rootPath)) { - const content = await fs.readFile(filePath, "utf8"); - findings.push(...scanContent(filePath, content, options.severity)); - } - return findings; -} - -function scanContent(filePath: string, content: string, severity: Severity): Finding[] { - const lines = content.split(/\r?\n/); - const found: Finding[] = []; - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i] ?? ""; - for (const pattern of PII_PATTERNS) { + return scanTextFilesByLine( + rootPath, + PII_PATTERNS, + (pattern, line) => { + // Patterns are global regexes, so reset lastIndex before each test. pattern.regex.lastIndex = 0; - if (pattern.regex.test(line)) { - found.push({ - file: filePath, - line: i + 1, - type: "pii", - severity, - source: "pii", - message: pattern.message, - term: pattern.name, - }); - } - } - } - return found; -} - -async function* walkTextFiles(rootPath: string): AsyncGenerator { - const stack: string[] = [rootPath]; - while (stack.length > 0) { - const current = stack.pop(); - if (!current) continue; - let entries; - try { - entries = await fs.readdir(current, { withFileTypes: true }); - } catch { - continue; - } - for (const entry of entries) { - const fullPath = path.join(current, entry.name); - if (entry.isDirectory()) { - if (SKIP_DIRECTORIES.has(entry.name)) continue; - stack.push(fullPath); - } else if (entry.isFile()) { - const ext = path.extname(entry.name).toLowerCase(); - if (BINARY_EXTENSIONS.has(ext)) continue; - const stat = await fs.stat(fullPath); - if (stat.size > 5 * 1024 * 1024) continue; - yield fullPath; - } - } - } + return pattern.regex.test(line); + }, + (pattern, filePath, lineNumber) => ({ + file: filePath, + line: lineNumber, + type: "pii", + severity: options.severity, + source: "pii", + message: pattern.message, + term: pattern.name, + }), + ); } diff --git a/tools/typescript/essentials-sync/src/scanners/text-files.ts b/tools/typescript/essentials-sync/src/scanners/text-files.ts new file mode 100644 index 0000000..f533320 --- /dev/null +++ b/tools/typescript/essentials-sync/src/scanners/text-files.ts @@ -0,0 +1,71 @@ +import { promises as fs } from "node:fs"; +import { globbyStream } from "globby"; +import type { Finding } from "../types.js"; + +// Always skipped, even when the scanned tree has no .gitignore. These are +// caches, build output, and VCS/dependency directories that never carry +// meaningful findings. +const HARD_SKIP_DIRECTORIES = [ + "node_modules", ".git", ".venv", "venv", "__pycache__", ".pytest_cache", + ".ruff_cache", ".mypy_cache", "dist", "build", ".pulumi", +]; + +const BINARY_EXTENSIONS = [ + ".png", ".jpg", ".jpeg", ".gif", ".webp", ".pdf", ".zip", ".tar", ".gz", + ".bz2", ".xz", ".7z", ".woff", ".woff2", ".eot", ".ttf", ".otf", ".ico", + ".so", ".dylib", ".dll", ".class", ".jar", ".wasm", ".bin", +]; + +const MAX_FILE_BYTES = 5 * 1024 * 1024; + +// Yields absolute paths of candidate text files under rootPath. Traversal, +// .gitignore handling, and directory/extension skipping are delegated to +// globby; the only thing globby cannot express is the per-file size cap, so +// oversized files are filtered with a stat after globbing. +export async function* walkTextFiles(rootPath: string): AsyncGenerator { + const ignore = [ + ...HARD_SKIP_DIRECTORIES.map((dir) => `**/${dir}/**`), + ...BINARY_EXTENSIONS.map((ext) => `**/*${ext}`), + "**/tmp-*", + "**/tmp.*", + ]; + const stream = globbyStream("**/*", { + cwd: rootPath, + absolute: true, + dot: true, + gitignore: true, + ignore, + }); + for await (const entry of stream) { + const filePath = entry as string; + const stat = await fs.stat(filePath); + if (stat.size > MAX_FILE_BYTES) continue; + yield filePath; + } +} + +// Runs a set of line-oriented patterns over every text file under rootPath. +// `test` decides whether a pattern matches a given line; `toFinding` builds the +// Finding for a match. This centralizes the read/split/loop that both the pii +// and jargon scanners would otherwise duplicate. +export async function scanTextFilesByLine

( + rootPath: string, + patterns: readonly P[], + test: (pattern: P, line: string) => boolean, + toFinding: (pattern: P, filePath: string, lineNumber: number) => Finding, +): Promise { + const findings: Finding[] = []; + for await (const filePath of walkTextFiles(rootPath)) { + const content = await fs.readFile(filePath, "utf8"); + const lines = content.split(/\r?\n/); + for (let i = 0; i < lines.length; i += 1) { + const line = lines[i] ?? ""; + for (const pattern of patterns) { + if (test(pattern, line)) { + findings.push(toFinding(pattern, filePath, i + 1)); + } + } + } + } + return findings; +} diff --git a/uv.lock b/uv.lock index 7cda940..108daae 100644 --- a/uv.lock +++ b/uv.lock @@ -9,6 +9,7 @@ members = [ "ess-auth", "ess-browser", "ess-dirs", + "ess-hello-jwt-auth-code", "ess-messages", "ess-outlook", "ess-service-now-incident", @@ -508,6 +509,33 @@ name = "ess-dirs" version = "0.1.0" source = { editable = "packages/python/ess-dirs" } +[[package]] +name = "ess-hello-jwt-auth-code" +version = "0.1.0" +source = { editable = "examples/python/ess-hello-jwt-auth-code" } +dependencies = [ + { name = "httpx" }, + { name = "python-dotenv" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "ruff" }, +] + +[package.metadata] +requires-dist = [ + { name = "httpx", specifier = ">=0.28.0" }, + { name = "python-dotenv", specifier = ">=1.1.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.0.0" }, + { name = "ruff", specifier = ">=0.14.2" }, +] + [[package]] name = "ess-messages" version = "0.1.0"