diff --git a/.azdo/ci-pr.yaml b/.azdo/ci-pr.yaml index 80fc4b4d..178875b0 100644 --- a/.azdo/ci-pr.yaml +++ b/.azdo/ci-pr.yaml @@ -43,7 +43,7 @@ extends: - script: | python -m pip install --upgrade pip python -m pip install flake8 black build diff-cover - python -m pip install -e .[dev] + python -m pip install -e .[dev,async] displayName: 'Install dependencies' - script: | diff --git a/.claude/skills/dataverse-sdk-use/SKILL.md b/.claude/skills/dataverse-sdk-use/SKILL.md index 72677468..d25815d7 100644 --- a/.claude/skills/dataverse-sdk-use/SKILL.md +++ b/.claude/skills/dataverse-sdk-use/SKILL.md @@ -28,11 +28,23 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` +- `len(result)` — number of records in this result/page ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +97,92 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) - -# Query with filter (paginated) -for page in client.records.get( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) + +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) +for record in result: + print(record["name"]) +``` + +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) + +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .order_by("name asc") + .page_size(500) + .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +255,24 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -223,6 +305,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetchxml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables @@ -380,7 +490,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +523,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +542,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -464,16 +576,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -486,9 +599,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 886bc72b..0178b090 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -30,7 +30,7 @@ jobs: run: | python -m pip install --upgrade pip python -m pip install flake8 black build diff-cover - python -m pip install -e .[dev] + python -m pip install -e .[dev,async] - name: Check format with black run: | diff --git a/CHANGELOG.md b/CHANGELOG.md index cf36ae04..3d0bbcce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,32 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added +- `client.records.retrieve(table, record_id, *, select, expand, include_annotations)` — fetch a single record by GUID; returns `None` on 404 instead of raising; `expand` adds `$expand` for navigation property expansion on the single-record GET; `include_annotations` maps to the `Prefer: odata.include-annotations` header for formatted values and lookup labels (#175) +- `client.records.list(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — eager fetch returning a flat `QueryResult`; GA replacement for `records.get()` without a record ID; `page_size` controls `Prefer: odata.maxpagesize`, `count=True` adds `$count=true`, `include_annotations` requests formatted values (#175) +- `client.records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — lazy iterator yielding one `QueryResult` per HTTP page; streaming counterpart to `list()`; same parameter set (#175) +- `client.query.fetchxml(xml)` — FetchXML support returning an inert `FetchXmlQuery`; no HTTP request is made until `.execute()` or `.execute_pages()` is called (#175) +- `FetchXmlQuery` implements the correct Dataverse paging cookie algorithm: annotation parsed as outer XML, `pagingcookie` attribute double URL-decoded, server-supplied `pagenumber` used for next page, `morerecords` handled as both `bool` and `"true"` string, `UserWarning` emitted on simple paging fallback, 32,768-character URL limit enforced (documented Dataverse GET cap), 10,000-page circuit breaker against runaway iteration (#175) +- `QueryBuilder.execute_pages()` — lazy per-page streaming returning one `QueryResult` per HTTP page; replaces deprecated `execute(by_page=True)` (#175) +- `QueryBuilder.where()` — composable filter expressions using `col()` and Python operators (`==`, `>`, `&`, `|`, `~`); replaces deprecated `filter_eq()`, `filter_contains()`, and other `filter_*` helpers (#175) +- `QueryResult.__getitem__` — index access (`result[0]`) returns a `Record`; slice access (`result[1:5]`) returns a new `QueryResult` (#175) +- `DataverseModel` structural `Protocol` (`models/protocol.py`) — implement on any entity class to enable typed integration with CRUD operations without specifying table names or serializing manually (#175) +- `col()`, `raw()`, `QueryResult`, and `DataverseModel` exported from the top-level `PowerPlatform.Dataverse` package (#175) +- v0→v1 migration tool: `tools/migrate_v0_to_v1.py` rewrites v0 call sites to the v1 API with `--dry-run` support; covers `create`, `update`, `delete`, `get`, `list`, `fetchxml`, and query builder patterns (#175) +- Migration tool now auto-rewrites `QueryBuilder.to_dataframe()` → `.execute().to_dataframe()` (inserts `.execute()` when receiver is a recognised builder chain); output improved with `[NEEDS-MANUAL]` label for files that have no auto-rewrites but require manual attention, and a trailing note on `[MIGRATED]` lines when manual items remain (#175) + +### Changed +- `QueryBuilder.execute()` now returns a flat `QueryResult` (all pages collected eagerly) instead of `Iterable[Record]` (#175) +- `records.get()` deprecation extended: calling with a `record_id` emits `DeprecationWarning` directing callers to `retrieve()`; calling without a `record_id` directs callers to `list()` (#175) + +### Deprecated +- `QueryBuilder.execute(by_page=True)` and `execute(by_page=False)` emit `UserWarning`; use `execute_pages()` and `execute()` respectively (#175) +- `client.query.odata_select()`, `client.query.odata_expands()`, `client.query.odata_expand()`, `client.query.odata_bind()` emit `DeprecationWarning`; navigation property helpers are replaced by `QueryBuilder.expand()` (#175) + +### Removed +- All v0 flat methods on `DataverseClient` (`create`, `update`, `delete`, `get`, `list`, `query_sql`, etc.) removed (~570 lines); use the `client.records`, `client.query`, and `client.batch` namespaces (#175) +- `client.query.sql_select()`, `client.query.sql_joins()`, `client.query.sql_join()` removed (#175) + ## [0.1.0b10] - 2026-05-12 ### Added diff --git a/README.md b/README.md index eab08734..43acb8f4 100644 --- a/README.md +++ b/README.md @@ -124,6 +124,7 @@ The SDK provides a simple, pythonic interface for Dataverse operations: | **Records** | Dataverse records represented as Python dictionaries with column schema names | | **Schema names** | Use table schema names (`"account"`, `"new_MyTestTable"`) and column schema names (`"name"`, `"new_MyTestColumn"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | | **Bulk Operations** | Efficient bulk processing for multiple records with automatic optimization | +| **QueryBuilder** | Preferred query API: `client.query.builder()` with composable `where(col(...))` filters, formatted values, expand, and streaming; use `records.list()` only as a shortcut for simple filter+select | | **Paging** | Automatic handling of large result sets with iterators | | **Structured Errors** | Detailed exception hierarchy with retry guidance and diagnostic information | | **Customization prefix values** | Custom tables and columns require a customization prefix value to be included for all operations (e.g., `"new_MyTestTable"`, not `"MyTestTable"`). See: [Table definitions in Microsoft Dataverse](https://learn.microsoft.com/en-us/power-apps/developer/data-platform/entity-metadata) | @@ -144,7 +145,7 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: contact_id = client.records.create("contact", {"firstname": "John", "lastname": "Doe"}) # Read the contact back - contact = client.records.get("contact", contact_id, select=["firstname", "lastname"]) + contact = client.records.retrieve("contact", contact_id, select=["firstname", "lastname"]) print(f"Created: {contact['firstname']} {contact['lastname']}") # Clean up @@ -159,9 +160,18 @@ with DataverseClient("https://yourorg.crm.dynamics.com", credential) as client: account_id = client.records.create("account", {"name": "Contoso Ltd"}) # Read a record -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(account["name"]) +# Read with expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + # Update a record client.records.update("account", account_id, {"telephone1": "555-0199"}) @@ -242,18 +252,25 @@ client.records.upsert("account", [ The SDK provides pandas wrappers for all CRUD operations via the `client.dataframe` namespace, using DataFrames and Series for input and output. +> **Note:** `client.dataframe.get()` is deprecated. Use the GA patterns shown below instead. + ```python import pandas as pd +from PowerPlatform.Dataverse.models.filters import col -# Query records as a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name", "telephone1"]) +# Query records as a single DataFrame (GA builder pattern) +df = (client.query.builder("account") + .select("name", "telephone1") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) print(f"Found {len(df)} accounts") # Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() # Fetch a single record as a one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -288,10 +305,12 @@ The **QueryBuilder** is the recommended way to query records. It provides a flue ```python # Fluent query builder (recommended) +from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .top(100) .page_size(50) @@ -299,47 +318,48 @@ for record in (client.query.builder("account") print(f"{record['name']}: {record['revenue']}") ``` -The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. All filter methods are discoverable via IDE autocomplete: +The QueryBuilder handles value formatting, column name casing, and OData syntax automatically. Filter expressions are built with `col()` and standard Python operators: ```python # Get results as a pandas DataFrame (consolidates all pages) df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) print(f"Got {len(df)} accounts") ``` ```python -# Comparison filters +# Comparison filters using col() expressions query = (client.query.builder("contact") - .filter_eq("statecode", 0) # statecode eq 0 - .filter_gt("revenue", 1000000) # revenue gt 1000000 - .filter_contains("name", "Corp") # contains(name, 'Corp') - .filter_in("statecode", [0, 1]) # Microsoft.Dynamics.CRM.In(...) - .filter_between("revenue", 100000, 500000) # (revenue ge 100000 and revenue le 500000) - .filter_null("telephone1") # telephone1 eq null + .where(col("statecode") == 0) # statecode eq 0 + .where(col("revenue") > 1000000) # revenue gt 1000000 + .where(col("name").contains("Corp")) # contains(name, 'Corp') + .where(col("statecode").in_([0, 1])) # Microsoft.Dynamics.CRM.In(...) + .where(col("revenue").between(100000, 500000)) # revenue ge 100000 and revenue le 500000 + .where(col("telephone1").is_null()) # telephone1 eq null ) ``` -For complex logic (OR, NOT, grouping), use the composable expression tree with `where()`: +For complex logic (OR, NOT, grouping), compose expressions with `&`, `|`, `~`: ```python -from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in, between +from PowerPlatform.Dataverse.models.filters import col # OR conditions: (statecode = 0 OR statecode = 1) AND revenue > 100k for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) + & (col("revenue") > 100000)) .execute()): print(record["name"]) # NOT, between, and in operators for record in (client.query.builder("account") - .where(~eq("statecode", 2)) # NOT inactive - .where(between("revenue", 100000, 500000)) # revenue in range + .where(col("statecode") != 2) # NOT inactive + .where(col("revenue").between(100000, 500000)) # revenue in range .execute()): print(record["name"]) ``` @@ -347,13 +367,31 @@ for record in (client.query.builder("account") **Formatted values and annotations** -- request localized labels, currency symbols, and display names: ```python -# Get formatted values (choice labels, currency, lookup names) +# Get formatted values (choice labels, currency, lookup names) — via query builder for record in (client.query.builder("account") .select("name", "statecode", "revenue") .include_formatted_values() .execute()): status = record["statecode@OData.Community.Display.V1.FormattedValue"] print(f"{record['name']}: {status}") + +# Get formatted values — via records.list() / records.retrieve() include_annotations param +result = client.records.list( + "account", + select=["name", "statecode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +for record in result: + label = record.get("statecode@OData.Community.Display.V1.FormattedValue") + print(f"{record['name']}: {label}") + +record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + include_annotations="OData.Community.Display.V1.FormattedValue", +) +if record: + print(record.get("statuscode@OData.Community.Display.V1.FormattedValue")) ``` **Nested expand with options** -- expand navigation properties with `$select`, `$filter`, `$orderby`, and `$top`: @@ -373,14 +411,83 @@ for record in (client.query.builder("account") print(record["name"], record.get("Account_Tasks")) ``` +**Paging** -- use `execute_pages()` for streaming large result sets with full builder options (filtering, sorting, formatted values). `records.list()` and `records.list_pages()` are simpler shortcuts for string-based OData filter queries: + +```python +# Preferred: query.builder().execute_pages() — stream one page at a time, memory stays flat +# Supports composable filters, sorting, formatted values, and expand with nested selects +for page_num, page in enumerate( + client.query.builder("account") + .select("accountid", "name", "revenue") + .where(col("statecode") == 0) + .order_by("name") + .page_size(500) # optional: override Dataverse default (~5000/page) + .execute_pages() +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(f" {record['name']}") + +# Simple shortcut: records.list() — automatic paging, all records in memory +# Use for basic filter+select queries; string OData filter only (no composable expressions) +result = client.records.list( + "account", + filter="statecode eq 0", + select=["name", "revenue"], + orderby=["name asc"], # optional sort + top=500, # bounds total records returned and number of HTTP round-trips + page_size=200, # optional: hint Dataverse default page size +) +for record in result: + print(record["name"]) + +# Simple streaming shortcut: records.list_pages() — same params as records.list(), yields one page at a time +for page_num, page in enumerate( + client.records.list_pages("account", filter="statecode eq 0", select=["name"], orderby=["name asc"]) +): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) +``` + +> **Deprecation note:** `execute(by_page=True)` and `execute(by_page=False)` are deprecated and emit a `UserWarning`. Replace with `execute_pages()` (streaming) or plain `execute()` (eager). `QueryBuilder.to_dataframe()` is also deprecated; use `.execute().to_dataframe()` instead. The migration tool (`tools/migrate_v0_to_v1.py`) rewrites all of these automatically. + **Record count** -- include `$count=true` in the request: ```python -# Request count alongside results +# Via query builder results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) + +# Via records.list() — count=True adds $count=true to the OData request +results = client.records.list("account", filter="statecode eq 0", count=True) +``` + +**FetchXML queries** -- `client.query.fetchxml()` returns an inert `FetchXmlQuery` object; no HTTP request is made until you call `.execute()` or `.execute_pages()`: + +```python +xml = """ + + + + + + + +""" + +# .execute() — blocking, fetches all pages and returns a single QueryResult +result = client.query.fetchxml(xml).execute() +df = result.to_dataframe() + +# .execute_pages() — streaming, yields one QueryResult per HTTP page +# Use count="N" in the FetchXML element to set page size +for page_num, page in enumerate(client.query.fetchxml(xml).execute_pages()): + print(f"Page {page_num + 1}: {len(page)} records") + for record in page: + print(record["name"]) ``` **SQL queries** provide an alternative read-only query syntax with support for @@ -405,52 +512,44 @@ df = client.dataframe.sql( "SELECT name, revenue FROM account ORDER BY revenue DESC" ) -# SQL helpers: discover columns and JOINs from metadata -cols = client.query.sql_select("account") # "accountid, name, revenue, ..." -join = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") -# Returns: "JOIN account a ON c.parentcustomerid = a.accountid" +# Discover columns from metadata (schema-discovery helper, kept at GA) +cols_meta = client.query.sql_columns("account") +col_names = [c["LogicalName"] for c in cols_meta] -# Build queries using helpers -- no OData knowledge needed -sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {join}" +# Build queries using the discovered column names +sql = f"SELECT TOP 10 {', '.join(col_names[:5])} FROM account" df = client.dataframe.sql(sql) - -# Discover all possible JOINs from a table (including polymorphic) -joins = client.query.sql_joins("opportunity") -for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") ``` -**Raw OData queries** are available via `records.get()` for cases where you need direct control over the OData filter string. The SDK provides helpers to eliminate the most error-prone parts: +**Simple list shortcut** -- `records.list()` accepts a raw OData filter string for basic queries. For anything beyond simple filter+select, prefer `client.query.builder()` (composable filters, formatted values, nested expand): ```python -# Discover columns for $select (returns list ready for select= parameter) -cols = client.query.odata_select("account") -for page in client.records.get("account", select=cols, top=10): - ... - -# Discover $expand navigation properties (auto-resolves PascalCase names) -nav = client.query.odata_expand("contact", "account") -# Returns: "parentcustomerid_account" -for page in client.records.get("contact", select=["fullname"], expand=[nav], top=5): - for r in page: - acct = r.get(nav) or {} - print(f"{r['fullname']} -> {acct.get('name')}") - -# Build @odata.bind for lookup fields (no manual name construction) -bind = client.query.odata_bind("contact", "account", account_id) -# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} -client.records.create("contact", {"firstname": "Jane", **bind}) - -# Raw OData query with manual parameters -for page in client.records.get( +# records.list() shortcut — raw OData filter string, all records loaded into memory +# Column names in filter must be lowercase logical names +for record in client.records.list( "account", select=["name"], - filter="statecode eq 0", # Raw OData: column names must be lowercase - expand=["primarycontactid"], # Navigation properties are case-sensitive + filter="statecode eq 0", top=100, ): - for record in page: - print(record["name"]) + print(record["name"]) + +# Discover navigation property names for $expand (metadata-discovery helper, kept at GA) +nav_props = client.query.odata_expands("account") # → list of navigation property metadata + +# Expand navigation properties using the query builder +from PowerPlatform.Dataverse.models.query_builder import ExpandOption +for record in (client.query.builder("contact") + .select("fullname") + .expand(ExpandOption("parentcustomerid_account").select("name")) + .execute()): + acct = record.get("parentcustomerid_account") or {} + print(f"{record['fullname']} -> {acct.get('name')}") + +# Build @odata.bind for lookup fields (deprecated helper, still functional with DeprecationWarning) +bind = client.query.odata_bind("contact", "account", account_id) +# Returns: {"parentcustomerid_account@odata.bind": "/accounts(guid)"} +client.records.create("contact", {"firstname": "Jane", **bind}) ``` ### Table management @@ -604,7 +703,14 @@ batch.records.create("account", {"name": "Contoso"}) batch.records.create("account", [{"name": "Fabrikam"}, {"name": "Woodgrove"}]) batch.records.update("account", account_id, {"telephone1": "555-0100"}) batch.records.delete("account", old_id) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"]) # single record with expand +batch.records.list( # multi-record, single page + "account", + filter="statecode eq 0", + select=["name"], + orderby=["name asc"], + top=50, +) result = batch.execute() for item in result.responses: @@ -718,7 +824,7 @@ from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import HttpError, ValidationError try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -742,9 +848,10 @@ For optimal performance in production environments: | Best Practice | Description | |---------------|-------------| +| **Prefer QueryBuilder for queries** | Use `client.query.builder()` for filtering, sorting, expansion, or formatted values; use `records.list()` only as a shortcut for simple filter+select | | **Bulk Operations** | Pass lists to `records.create()`, `records.update()` for automatic bulk processing, for `records.delete()`, set `use_bulk_delete` when passing lists to use bulk operation | | **Select Fields** | Specify `select` parameter to limit returned columns and reduce payload size | -| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage | +| **Page Size Control** | Use `top` and `page_size` parameters to control memory usage; use `execute_pages()` for large result sets | | **Connection Reuse** | Reuse `DataverseClient` instances across operations | | **Production Credentials** | Use `ClientSecretCredential` or `CertificateCredential` for unattended operations | | **Error Handling** | Implement retry logic for transient errors (`e.is_transient`) | diff --git a/examples/advanced/alternate_keys_upsert.py b/examples/advanced/alternate_keys_upsert.py index 67e8a43e..3248282a 100644 --- a/examples/advanced/alternate_keys_upsert.py +++ b/examples/advanced/alternate_keys_upsert.py @@ -100,29 +100,37 @@ def main(): client = DataverseClient(base_url, credential) # ------------------------------------------------------------------ - # Step 1: Create table + # Step 1: Create table (skip if already exists) # ------------------------------------------------------------------ print("\n1. Creating table...") - table_info = backoff( - lambda: client.tables.create( - TABLE_NAME, - columns={ - KEY_COLUMN: "string", - "new_ProductName": "string", - "new_Price": "decimal", - }, + table_info = client.tables.get(TABLE_NAME) + if table_info: + print(f" Table already exists: {TABLE_NAME} (skipped)") + else: + table_info = backoff( + lambda: client.tables.create( + TABLE_NAME, + columns={ + KEY_COLUMN: "string", + "new_ProductName": "string", + "new_Price": "decimal", + }, + ) ) - ) - print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") - - time.sleep(10) # Wait for metadata propagation + print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") + time.sleep(10) # Wait for metadata propagation # ------------------------------------------------------------------ - # Step 2: Create alternate key + # Step 2: Create alternate key (skip if already exists) # ------------------------------------------------------------------ print("\n2. Creating alternate key...") - key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) - print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") + existing_keys = client.tables.get_alternate_keys(TABLE_NAME) + existing_key = next((k for k in existing_keys if k.schema_name == KEY_NAME), None) + if existing_key: + print(f" Alternate key already exists: {KEY_NAME} (skipped)") + else: + key_info = backoff(lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()])) + print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") # ------------------------------------------------------------------ # Step 3: Wait for key to become Active @@ -212,15 +220,14 @@ def main(): # Step 6: Verify # ------------------------------------------------------------------ print("\n6. Verifying records...") - for page in client.records.get( + for record in client.records.list( TABLE_NAME, select=["new_productname", "new_price", KEY_COLUMN.lower()], ): - for record in page: - ext_id = record.get(KEY_COLUMN.lower(), "?") - name = record.get("new_productname", "?") - price = record.get("new_price", "?") - print(f" {ext_id}: {name} @ ${price}") + ext_id = record.get(KEY_COLUMN.lower(), "?") + name = record.get("new_productname", "?") + price = record.get("new_price", "?") + print(f" {ext_id}: {name} @ ${price}") # ------------------------------------------------------------------ # Step 7: List alternate keys diff --git a/examples/advanced/batch.py b/examples/advanced/batch.py index a95aa303..bb180bfc 100644 --- a/examples/advanced/batch.py +++ b/examples/advanced/batch.py @@ -13,248 +13,258 @@ from __future__ import annotations -# --------------------------------------------------------------------------- -# Setup — replace with your environment URL and credential -# --------------------------------------------------------------------------- +import sys from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient -credential = InteractiveBrowserCredential() -with DataverseClient("https://org.crm.dynamics.com", credential) as client: +def main(): + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") - # --------------------------------------------------------------------------- - # Example 1: Record CRUD in a single batch - # --------------------------------------------------------------------------- + credential = InteractiveBrowserCredential() - print("\n[INFO] Example 1: Record CRUD in a single batch") + with DataverseClient(base_url=base_url, credential=credential) as client: - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- - # Create a single record - batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + print("\n[INFO] Example 1: Record CRUD in a single batch") - # Create multiple records via CreateMultiple (one batch item) - batch.records.create( - "contact", - [ - {"firstname": "Alice", "lastname": "Smith"}, - {"firstname": "Bob", "lastname": "Jones"}, - ], - ) + batch = client.batch.new() - # Assume we have an existing account_id from a prior operation - # batch.records.update("account", account_id, {"telephone1": "555-9999"}) - # batch.records.delete("account", old_id) + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) - result = batch.execute() + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) - print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for guid in result.entity_ids: - print(f"[OK] Created: {guid}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) - # --------------------------------------------------------------------------- - # Example 2: Transactional changeset with content-ID chaining - # --------------------------------------------------------------------------- + result = batch.execute() - print("\n[INFO] Example 2: Transactional changeset") + print(f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") - batch = client.batch.new() + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- - with batch.changeset() as cs: - # Each create() returns a "$n" reference usable in subsequent operations - lead_ref = cs.records.create( - "lead", - {"firstname": "Ada", "lastname": "Lovelace"}, - ) - contact_ref = cs.records.create("contact", {"firstname": "Ada"}) - - # Reference the newly created lead and contact in the account - cs.records.create( - "account", - { - "name": "Babbage & Co.", - "originatingleadid@odata.bind": lead_ref, - "primarycontactid@odata.bind": contact_ref, - }, + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", ) - # Update using a content-ID reference as the record_id - cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + # Read table metadata + batch.tables.get("new_Product") - result = batch.execute() + # List all non-private tables + batch.tables.list() - if result.has_errors: - print("[ERR] Changeset rolled back") + result = batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.get("account", account_id, select=["name", "statecode"]) + # + # result = batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.retrieve("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") for item in result.failed: - print(f" {item.status_code}: {item.error_message}") - else: - print(f"[OK] {len(result.entity_ids)} records created atomically") - - # --------------------------------------------------------------------------- - # Example 3: Table metadata operations in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 3: Table metadata operations") - - batch = client.batch.new() - - # Create a new custom table - batch.tables.create( - "new_Product", - {"new_Price": "decimal", "new_InStock": "bool"}, - solution="MySolution", - ) - - # Read table metadata - batch.tables.get("new_Product") - - # List all non-private tables - batch.tables.list() - - result = batch.execute() - print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") - - # --------------------------------------------------------------------------- - # Example 4: SQL query in a batch - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 4: SQL query in batch") - - batch = client.batch.new() - batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") - - result = batch.execute() - if result.responses and result.responses[0].is_success and result.responses[0].data: - rows = result.responses[0].data.get("value", []) - print(f"[OK] Retrieved {len(rows)} accounts") - for row in rows: - print(f" {row.get('name')}") - - # --------------------------------------------------------------------------- - # Example 5: Mixed batch — changeset writes + standalone GETs - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 5: Mixed batch") - - # NOTE: Commented out because it requires a pre-existing account_id. - # Uncomment and set account_id to run this example. - # batch = client.batch.new() - # - # with batch.changeset() as cs: - # cs.records.update("account", account_id, {"statecode": 0}) - # - # batch.records.get("account", account_id, select=["name", "statecode"]) - # - # result = batch.execute() - # update_response = result.responses[0] - # account_data = result.responses[1] - # if account_data.is_success and account_data.data: - # print(f"Account: {account_data.data.get('name')}") - - # --------------------------------------------------------------------------- - # Example 6: Continue on error - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 6: Continue on error") - - batch = client.batch.new() - batch.records.get("account", "00000000-0000-0000-0000-000000000000") - batch.query.sql("SELECT TOP 1 name FROM account") - - result = batch.execute(continue_on_error=True) - print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") - for item in result.failed: - print(f"[ERR] {item.status_code}: {item.error_message}") - - # --------------------------------------------------------------------------- - # Example 7: DataFrame integration - # --------------------------------------------------------------------------- - - print("\n[INFO] Example 7: DataFrame batch operations") - - import pandas as pd - - # Create records from a DataFrame - df = pd.DataFrame( - [ - {"name": "DF-Batch-A", "telephone1": "555-0100"}, - {"name": "DF-Batch-B", "telephone1": "555-0200"}, - ] - ) - batch = client.batch.new() - batch.dataframe.create("account", df) - result = batch.execute() - print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") - created_ids = list(result.entity_ids) - - # Update records from a DataFrame - if len(created_ids) >= 2: - update_df = pd.DataFrame( + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( [ - {"accountid": created_ids[0], "telephone1": "555-9990"}, - {"accountid": created_ids[1], "telephone1": "555-9991"}, + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, ] ) batch = client.batch.new() - batch.dataframe.update("account", update_df, id_column="accountid") + batch.dataframe.create("account", df) result = batch.execute() - print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: - # Delete records from a Series - if created_ids: batch = client.batch.new() - batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) - result = batch.execute() - print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") - - # --------------------------------------------------------------------------- - # Example 8: Understanding response data patterns - # --------------------------------------------------------------------------- + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") - print("\n[INFO] Example 8: Response data patterns") + result = batch.execute() - # Every batch result maps 1:1 with the operations you added. - # Different operations return different response shapes: + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue - batch = client.batch.new() - # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header - batch.records.create("account", {"name": "Pattern-Demo"}) - # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} - batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) - # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} - batch.query.sql("SELECT TOP 3 name FROM account") + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") - result = batch.execute() + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") - for i, resp in enumerate(result.responses): - if not resp.is_success: - print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") - continue + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Single create: entity_id from OData-EntityId header - if resp.entity_id: - print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") - # Bulk action (CreateMultiple/UpsertMultiple): IDs in body - elif resp.data and "Ids" in resp.data: - print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + # Clean up demo records + for rid in result.entity_ids: + client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + client.records.delete("account", rid) - # Query: rows in body - elif resp.data and "value" in resp.data: - print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") - # Delete or metadata operation: 204, no data - else: - print(f" Op {i}: [OK] {resp.status_code}") - - # Clean up demo records - for rid in result.entity_ids: - client.records.delete("account", rid) - for resp in result.succeeded: - if resp.data and "Ids" in resp.data: - for rid in resp.data["Ids"]: - client.records.delete("account", rid) +if __name__ == "__main__": + main() diff --git a/examples/advanced/dataframe_operations.py b/examples/advanced/dataframe_operations.py index 7c0b6010..0a51b4c7 100644 --- a/examples/advanced/dataframe_operations.py +++ b/examples/advanced/dataframe_operations.py @@ -19,6 +19,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw def main(): @@ -81,7 +82,7 @@ def _run_walkthrough(client): print("2. Query records as a DataFrame") print("-" * 60) - df_all = client.dataframe.get(table, select=select_cols, filter=test_filter) + df_all = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f"[OK] Got {len(df_all)} records in one DataFrame") print(f" Columns: {list(df_all.columns)}") print(f"{df_all.to_string(index=False)}") @@ -91,7 +92,7 @@ def _run_walkthrough(client): print("3. Limit results with top") print("-" * 60) - df_top2 = client.dataframe.get(table, select=select_cols, filter=test_filter, top=2) + df_top2 = client.query.builder(table).select(*select_cols).where(raw(test_filter)).top(2).execute().to_dataframe() print(f"[OK] Got {len(df_top2)} records with top=2") print(f"{df_top2.to_string(index=False)}") @@ -102,7 +103,9 @@ def _run_walkthrough(client): first_id = new_accounts["accountid"].iloc[0] print(f" Fetching record {first_id}...") - single = client.dataframe.get(table, record_id=first_id, select=select_cols) + single = ( + client.query.builder(table).select(*select_cols).where(col("accountid") == first_id).execute().to_dataframe() + ) print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") # -- 5. Update records from a DataFrame ------------------------ @@ -116,7 +119,7 @@ def _run_walkthrough(client): print("[OK] Updated 3 records") # Verify the updates - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # -- 6. Broadcast update (same value to all records) ----------- @@ -131,7 +134,7 @@ def _run_walkthrough(client): print("[OK] Broadcast update complete") # Verify all records have the same websiteurl - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified:\n{verified.to_string(index=False)}") # Default: NaN/None fields are skipped (not overridden on server) @@ -142,14 +145,14 @@ def _run_walkthrough(client): ] ) client.dataframe.update(table, sparse_df, id_column="accountid") - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n{verified.to_string(index=False)}") # Opt-in: clear_nulls=True sends None as null to clear the field print("\n Clearing websiteurl for Contoso with clear_nulls=True...") clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) - verified = client.dataframe.get(table, select=select_cols, filter=test_filter) + verified = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified (Contoso websiteurl should be empty):\n{verified.to_string(index=False)}") # -- 7. Delete records by passing a Series of GUIDs ------------ @@ -162,7 +165,7 @@ def _run_walkthrough(client): print(f"[OK] Deleted {len(new_accounts)} records") # Verify deletions - filter for our tagged records should return 0 - remaining = client.dataframe.get(table, select=select_cols, filter=test_filter) + remaining = client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute().to_dataframe() print(f" Verified: {len(remaining)} test records remaining (expected 0)") print("\n" + "=" * 60) diff --git a/examples/advanced/datascience_risk_assessment.py b/examples/advanced/datascience_risk_assessment.py index 338713e4..80dafdc4 100644 --- a/examples/advanced/datascience_risk_assessment.py +++ b/examples/advanced/datascience_risk_assessment.py @@ -50,6 +50,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw # -- Optional imports (graceful degradation if not installed) ------ @@ -272,43 +273,42 @@ def step1_extract(client): print("=" * 60) # Pull accounts - accounts = client.dataframe.get( - TABLE_ACCOUNTS, - select=["accountid", "name", "revenue", "numberofemployees", "industrycode"], - filter="statecode eq 0", - top=200, + accounts = ( + client.query.builder(TABLE_ACCOUNTS) + .select("accountid", "name", "revenue", "numberofemployees", "industrycode") + .where(col("statecode") == 0) + .top(200) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(accounts)} active accounts") # Pull open cases (service incidents) - cases = client.dataframe.get( - TABLE_CASES, - select=[ - "incidentid", - "_customerid_value", - "title", - "severitycode", - "prioritycode", - "createdon", - ], - filter="statecode eq 0", - top=1000, + cases = ( + client.query.builder(TABLE_CASES) + .select("incidentid", "_customerid_value", "title", "severitycode", "prioritycode", "createdon") + .where(raw("statecode eq 0")) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(cases)} open cases") # Pull active opportunities - opportunities = client.dataframe.get( - TABLE_OPPORTUNITIES, - select=[ + opportunities = ( + client.query.builder(TABLE_OPPORTUNITIES) + .select( "opportunityid", "_parentaccountid_value", "name", "estimatedvalue", "closeprobability", "estimatedclosedate", - ], - filter="statecode eq 0", - top=1000, + ) + .where(col("statecode") == 0) + .top(1000) + .execute() + .to_dataframe() ) print(f"[OK] Extracted {len(opportunities)} active opportunities") diff --git a/examples/advanced/fetchxml.py b/examples/advanced/fetchxml.py new file mode 100644 index 00000000..d4ac1e50 --- /dev/null +++ b/examples/advanced/fetchxml.py @@ -0,0 +1,582 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +End-to-end FetchXML examples for Dataverse. + +Demonstrates ``client.query.fetchxml()`` across the scenarios where FetchXML +is required or preferred over OData/SQL: + +- Basic attribute queries +- operators (eq, like, in, null, not-null, between) +- (inner and outer joins) +- Ordering +- Top N with automatic paging-cookie propagation +- Aggregate queries (count, sum, avg, min, max, group-by) +- Built-in system tables (account → contact join) + +FetchXML is the right tool when: +- You need a JOIN type OData $expand cannot express (many-to-many, outer link) +- You need server-side aggregates (count, sum, avg) without GROUP BY SQL +- You need ```` operators unavailable in OData ($filter) + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import sys +import time + +from azure.identity import InteractiveBrowserCredential +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError +import requests + +# --------------------------------------------------------------------------- +# Helpers (same pattern as sql_examples.py) +# --------------------------------------------------------------------------- + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +def backoff(op, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + time.sleep(d) + total_delay += d + attempts += 1 + try: + result = op() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +def main(): + print("=" * 80) + print("Dataverse SDK -- FetchXML End-to-End Examples") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("InteractiveBrowserCredential()") + credential = InteractiveBrowserCredential() + + log_call(f"DataverseClient(base_url='{base_url}', credential=...)") + with DataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + _run_examples(client) + + +def _run_examples(client): + project_table = "new_FXDemoProject" + task_table = "new_FXDemoTask" + + # =================================================================== + # 2. Create tables and seed data + # =================================================================== + heading(2, "Create Tables & Seed Data") + + log_call(f"client.tables.get('{project_table}')") + if client.tables.get(project_table): + print(f"[OK] Table already exists: {project_table}") + else: + log_call(f"client.tables.create('{project_table}', ...)") + try: + backoff( + lambda: client.tables.create( + project_table, + { + "new_Code": "string", + "new_Budget": "decimal", + "new_Active": "bool", + "new_Region": "int", + }, + ) + ) + print(f"[OK] Created table: {project_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {project_table} (skipped)") + else: + raise + + log_call(f"client.tables.get('{task_table}')") + if client.tables.get(task_table): + print(f"[OK] Table already exists: {task_table}") + else: + log_call(f"client.tables.create('{task_table}', ...)") + try: + backoff( + lambda: client.tables.create( + task_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {task_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {task_table} (skipped)") + else: + raise + + print("\n[INFO] Creating lookup field: tasks → projects ...") + try: + client.tables.create_lookup_field( + referencing_table=task_table, + lookup_field_name="new_ProjectId", + referenced_table=project_table, + display_name="Project", + ) + print("[OK] Created lookup: new_ProjectId on tasks → projects") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + # Resolve entity set name for @odata.bind + project_set = f"{project_table.lower()}s" + try: + tinfo = client.tables.get(project_table) + if tinfo: + project_set = tinfo.get("entity_set_name", project_set) + except Exception: + pass + + log_call(f"client.records.create('{project_table}', [...])") + projects = [ + {"new_Code": "ALPHA", "new_Budget": 50000, "new_Active": True, "new_Region": 1}, + {"new_Code": "BRAVO", "new_Budget": 75000, "new_Active": True, "new_Region": 2}, + {"new_Code": "CHARLIE", "new_Budget": 30000, "new_Active": False, "new_Region": 3}, + {"new_Code": "DELTA", "new_Budget": 90000, "new_Active": True, "new_Region": 1}, + {"new_Code": "ECHO", "new_Budget": 42000, "new_Active": True, "new_Region": 2}, + ] + project_ids = backoff(lambda: client.records.create(project_table, projects)) + print(f"[OK] Seeded {len(project_ids)} projects") + + log_call(f"client.records.create('{task_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[4]})", + }, + ] + task_ids = backoff(lambda: client.records.create(task_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks") + + project_logical = project_table.lower() # new_fxdemoproject + task_logical = task_table.lower() # new_fxdemotask + project_pk = f"{project_logical}id" # new_fxdemoprojectid + task_pk = f"{task_logical}id" # new_fxdemoproject id + lookup_attr = "new_projectid" # lookup logical name on task + + try: + # =============================================================== + # 3. Basic attribute query + # =============================================================== + heading(3, "Basic Attribute Query") + xml = f""" + + + + + + + + """ + log_call("client.query.fetchxml(basic attribute query)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} projects:") + for r in result: + print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + # Index access and first() are equivalent; first() returns None on empty result + if result: + print(f" First by index : {result[0].get('new_code')}") + print(f" First by .first(): {result.first().get('new_code')}") + + # =============================================================== + # 4. operators: eq, like, in, null, not-null, between + # =============================================================== + heading(4, " Operators") + + # eq + xml = f""" + + + + + + + + + """ + log_call('operator="eq" value="ALPHA"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] eq: {[x.get('new_code') for x in r]}") + + # like + xml = f""" + + + + + + + + + """ + log_call('operator="like" value="%test%"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") + + # in + xml = f""" + + + + + + ALPHA + DELTA + + + + + """ + log_call('operator="in" values=[ALPHA, DELTA]') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] in: {[x.get('new_code') for x in r]}") + + # null / not-null + xml = f""" + + + + + + + + + """ + log_call('operator="not-null"') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] not-null: {len(r)} tasks have priority set") + + # between + xml = f""" + + + + + + + 40000 + 80000 + + + + + """ + log_call('operator="between" 40000 and 80000') + r = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") + + # =============================================================== + # 5. — inner join (tasks → projects) + # =============================================================== + heading(5, " Inner Join (Tasks → Projects)") + xml = f""" + + + + + + + + + + + """ + log_call("client.query.fetchxml(link-entity inner join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows:") + for r in result: + print( + f" Task={r.get('new_title', ''):<25s} " + f"Hours={r.get('new_hours')} " + f"Project={r.get('p.new_code', '')} " + f"Budget={r.get('p.new_budget')}" + ) + except Exception as e: + print(f"[WARN] link-entity join failed: {e}") + + # =============================================================== + # 6. — outer join (projects with or without tasks) + # =============================================================== + heading(6, " Outer Join (Projects With or Without Tasks)") + xml = f""" + + + + + + + + + """ + log_call("client.query.fetchxml(link-entity outer join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows (includes projects with no tasks):") + for r in result[:8]: + print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") + except Exception as e: + print(f"[WARN] outer join failed: {e}") + + # =============================================================== + # 7. Ordering + # =============================================================== + heading(7, "Ordering ( element)") + + xml = f""" + + + + + + + + """ + log_call("client.query.fetchxml(order by hours DESC)") + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Tasks by hours DESC:") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + # =============================================================== + # 8. Top N + paging-cookie propagation + # =============================================================== + heading(8, "Paging-Cookie Propagation") + print( + "[INFO] 'count' sets the page size in FetchXML (not 'top' — 'top' is a total-result limit).\n" + "With count='2' and 7 seeded tasks the server returns pages of 2, 2, 2, 1.\n" + ".execute() collects all pages eagerly; .execute_pages() yields one QueryResult per HTTP page." + ) + xml_paged = f""" + + + + + + + + """ + log_call("client.query.fetchxml(xml).execute() — eager, all pages collected") + result = backoff(lambda: client.query.fetchxml(xml_paged).execute()) + print(f"[OK] execute(): {len(result)} total tasks across all pages (seeded {len(task_ids)}):") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + log_call("client.query.fetchxml(xml).execute_pages() — lazy, one QueryResult per HTTP page") + page_num = 0 + page_record_count = 0 + for page in backoff(lambda: client.query.fetchxml(xml_paged).execute_pages()): + page_num += 1 + page_record_count += len(page) + print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") + print(f"[OK] execute_pages(): {page_record_count} total tasks across {page_num} page(s)") + + # =============================================================== + # 9. Aggregates (count, sum, avg, min, max) + # =============================================================== + heading(9, "Aggregate Queries ()") + + # Global aggregates + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetchxml(aggregate: count, sum, avg, min, max)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + if result: + row = result.first() + print( + f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " + f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" + ) + except Exception as e: + print(f"[WARN] aggregate failed: {e}") + + # Group-by aggregate: total hours per project + xml = f""" + + + + + + + + + + """ + log_call("client.query.fetchxml(aggregate group-by project)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Hours per project ({len(result)} groups):") + for r in result: + print( + f" {r.get('project_code', ''):<10s} " + f"Tasks={r.get('task_count')} " + f"Hours={r.get('total_hours')}" + ) + except Exception as e: + print(f"[WARN] group-by aggregate failed: {e}") + + # =============================================================== + # 10. Built-in system tables (account → contact) + # =============================================================== + heading(10, "Built-In System Tables (account → contact Join)") + xml = """ + + + + + + + + + """ + log_call("client.query.fetchxml(account → contact inner join)") + try: + result = backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} account-contact pairs:") + for r in result: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") + except Exception as e: + print(f"[INFO] No account-contact data in this org: {e}") + + finally: + heading(11, "Cleanup") + for tbl in [task_table, project_table]: + log_call(f"client.tables.delete('{tbl}')") + try: + backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + code = getattr(getattr(ex, "response", None), "status_code", None) + if isinstance(ex, (requests.exceptions.HTTPError, MetadataError)) and code == 404: + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("FetchXML Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + main() diff --git a/examples/advanced/prodev_quick_start.py b/examples/advanced/prodev_quick_start.py index e28d1575..d06e058f 100644 --- a/examples/advanced/prodev_quick_start.py +++ b/examples/advanced/prodev_quick_start.py @@ -56,6 +56,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.filters import col # -- Table schema names -- # Uses the standard 'new_' publisher prefix (default Dataverse publisher). @@ -115,7 +116,7 @@ def run_demo(client): customer_ids, project_ids, task_ids = step3_populate_data(client, primary_name_col) # -- Step 4: Query and analyze -- - step4_query_and_analyze(client, customer_ids, primary_name_col) + step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col) # -- Step 5: Update and delete -- step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) @@ -298,10 +299,11 @@ def step3_populate_data(client, primary_name_col): print(f"[OK] Created {len(customers_df)} customers") # -- Projects (linked to customers via lookup) -- - # @odata.bind keys use the navigation property logical name (lowercase) - # and the entity set name (also lowercase) in the value. - customer_lookup = f"{TABLE_PROJECT}_CustomerId".lower() + "@odata.bind" - customer_set = TABLE_CUSTOMER.lower() + "s" + # @odata.bind keys use the lookup field schema name (case-sensitive) + # and the entity set name (from table metadata) in the value. + customer_lookup = f"{TABLE_PROJECT}_CustomerId@odata.bind" + customer_info = client.tables.get(TABLE_CUSTOMER) + customer_set = customer_info.get("entity_set_name") if customer_info else TABLE_CUSTOMER.lower() + "s" projects_df = pd.DataFrame( [ { @@ -352,8 +354,9 @@ def step3_populate_data(client, primary_name_col): for i, (task_name, priority, status, hours) in enumerate(task_names): proj_idx = project_assignment[i] - project_lookup = f"{TABLE_TASK}_ProjectId".lower() + "@odata.bind" - project_set = TABLE_PROJECT.lower() + "s" + project_lookup = f"{TABLE_TASK}_ProjectId@odata.bind" + project_info = client.tables.get(TABLE_PROJECT) + project_set = project_info.get("entity_set_name") if project_info else TABLE_PROJECT.lower() + "s" tasks_data.append( { name_col: task_name, @@ -382,7 +385,7 @@ def step3_populate_data(client, primary_name_col): # ================================================================ -def step4_query_and_analyze(client, customer_ids, primary_name_col): +def step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col): """Query data and demonstrate DataFrame analysis.""" print("\n" + "-" * 60) print("STEP 4: Query and analyze data") @@ -392,26 +395,21 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Note: The SDK lowercases $select values automatically, so schema-name # casing (e.g., new_DemoProject_Budget) works -- it becomes the logical name. name_attr = primary_name_col - projects = client.dataframe.get( - TABLE_PROJECT, - select=[ - name_attr, - f"{TABLE_PROJECT}_Budget", - f"{TABLE_PROJECT}_Status", - ], + projects = ( + client.query.builder(TABLE_PROJECT) + .select(name_attr, f"{TABLE_PROJECT}_Budget", f"{TABLE_PROJECT}_Status") + .execute() + .to_dataframe() ) print(f"\n All projects ({len(projects)} rows):") print(f"{projects.to_string(index=False)}") # Query tasks and analyze - tasks = client.dataframe.get( - TABLE_TASK, - select=[ - name_attr, - f"{TABLE_TASK}_Priority", - f"{TABLE_TASK}_Status", - f"{TABLE_TASK}_EstimatedHours", - ], + tasks = ( + client.query.builder(TABLE_TASK) + .select(name_attr, f"{TABLE_TASK}_Priority", f"{TABLE_TASK}_Status", f"{TABLE_TASK}_EstimatedHours") + .execute() + .to_dataframe() ) print(f"\n All tasks ({len(tasks)} rows):") print(f"{tasks.to_string(index=False)}") @@ -440,7 +438,7 @@ def step4_query_and_analyze(client, customer_ids, primary_name_col): # Fetch single record by ID first_id = customer_ids.iloc[0] - single = client.dataframe.get(TABLE_CUSTOMER, record_id=first_id) + single = client.query.builder(TABLE_CUSTOMER).where(col(primary_id_col) == first_id).execute().to_dataframe() print(f"\n Single customer record (by ID):") print(f"{single.to_string(index=False)}") @@ -481,10 +479,7 @@ def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): print(f"[OK] Deleted 1 task") # Verify - remaining = client.dataframe.get( - TABLE_TASK, - select=[primary_name_col, status_col], - ) + remaining = client.query.builder(TABLE_TASK).select(primary_name_col, status_col).execute().to_dataframe() print(f"\n Remaining tasks ({len(remaining)}):") print(f"{remaining.to_string(index=False)}") diff --git a/examples/advanced/sql_examples.py b/examples/advanced/sql_examples.py index d9fd9627..372a3567 100644 --- a/examples/advanced/sql_examples.py +++ b/examples/advanced/sql_examples.py @@ -22,7 +22,7 @@ - SQL read -> DataFrame transform -> SDK write-back (full round-trip) - AND/OR, NOT IN, NOT LIKE boolean logic - Deep JOINs (5-8 tables) with no server depth limit -- SQL helper functions: sql_columns, sql_select, sql_joins, sql_join +- SQL helper functions: sql_columns (sql_select/sql_join/sql_joins removed at GA) - OData helper functions: odata_select, odata_expands, odata_expand, odata_bind - SQL vs OData side-by-side comparison @@ -133,58 +133,66 @@ def _run_examples(client): ) log_call(f"client.tables.get('{parent_table}')") - info = backoff(lambda: client.tables.get(parent_table)) - if info: + if client.tables.get(parent_table): print(f"[OK] Table already exists: {parent_table}") else: log_call(f"client.tables.create('{parent_table}', ...)") - info = backoff( - lambda: client.tables.create( - parent_table, - { - "new_Code": "string", - "new_Region": Region, - "new_Budget": "decimal", - "new_Active": "bool", - }, + try: + backoff( + lambda: client.tables.create( + parent_table, + { + "new_Code": "string", + "new_Region": Region, + "new_Budget": "decimal", + "new_Active": "bool", + }, + ) ) - ) - print(f"[OK] Created table: {parent_table}") + print(f"[OK] Created table: {parent_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {parent_table} (skipped)") + else: + raise log_call(f"client.tables.get('{child_table}')") - info2 = backoff(lambda: client.tables.get(child_table)) - if info2: + if client.tables.get(child_table): print(f"[OK] Table already exists: {child_table}") else: log_call(f"client.tables.create('{child_table}', ...)") - info2 = backoff( - lambda: client.tables.create( - child_table, - { - "new_Title": "string", - "new_Hours": "int", - "new_Done": "bool", - "new_Priority": "int", - }, + try: + backoff( + lambda: client.tables.create( + child_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) ) - ) - print(f"[OK] Created table: {child_table}") + print(f"[OK] Created table: {child_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {child_table} (skipped)") + else: + raise # Create lookup so tasks reference teams via JOIN print("\n[INFO] Creating lookup field so tasks reference teams via JOIN...") try: - backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_table, - lookup_field_name="new_TeamId", - referenced_table=parent_table, - display_name="Team", - ) + client.tables.create_lookup_field( + referencing_table=child_table, + lookup_field_name="new_TeamId", + referenced_table=parent_table, + display_name="Team", ) print("[OK] Created lookup: new_TeamId on tasks -> teams") except Exception as e: msg = str(e).lower() - if "already exists" in msg or "duplicate" in msg: + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: print("[OK] Lookup already exists (skipped)") else: raise @@ -419,19 +427,14 @@ def _run_examples(client): heading(13, "SQL -- INNER JOIN") print("Use the lookup attribute's logical name (e.g. new_teamid) for JOINs.") - # Use sql_join() to auto-discover the relationship and build - # the JOIN clause with proper aliases. + # Build the JOIN clause manually (sql_join() was removed at GA). + # Use the lookup attribute's logical name (not _..._value) as the join column. lookup_col = "new_teamid" # Lookup logical name, NOT _..._value - join_clause = client.query.sql_join( - from_table=child_table, - to_table=parent_table, - from_alias="tk", - to_alias="t", - ) + join_clause = f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_logical}id" print(f"[INFO] Lookup column: {lookup_col}") - print(f"[INFO] Generated JOIN: {join_clause}") + print(f"[INFO] JOIN clause: {join_clause}") - sql = f"SELECT t.new_code, tk.new_title, tk.new_hours " f"FROM {child_table} tk " f"{join_clause}" + sql = f"SELECT t.new_code, tk.new_title, tk.new_hours FROM {child_table} tk {join_clause}" log_call('client.query.sql("...INNER JOIN...")') try: results = backoff(lambda: client.query.sql(sql)) @@ -912,71 +915,26 @@ def _run_examples(client): # ============================================================== heading(29, "SQL Helper Functions (query.sql_*)") print( - "The SDK provides helper functions that auto-discover column\n" - "names and JOIN clauses from metadata -- no guessing needed." + "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" + "sql_select(), sql_join(), and sql_joins() were removed — write JOIN\n" + "clauses directly or use client.query.fetchxml() for complex queries." ) - # sql_columns + # sql_columns — still available at GA log_call(f"client.query.sql_columns('{parent_table}')") cols = client.query.sql_columns(parent_table) print(f"[OK] {len(cols)} columns:") for c in cols[:5]: print(f" {c['name']:30s} Type: {c['type']:15s} PK={c['is_pk']}") - # sql_select - log_call(f"client.query.sql_select('{parent_table}')") - select_str = client.query.sql_select(parent_table) - print(f"[OK] SELECT list: {select_str[:60]}...") - - # sql_joins - log_call(f"client.query.sql_joins('{child_table}')") - joins = client.query.sql_joins(child_table) - print(f"[OK] {len(joins)} possible JOINs:") - for j in joins[:5]: - print(f" {j['column']:25s} -> {j['target']}.{j['target_pk']}") - - # sql_joins -- alias uniqueness: multiple lookups to the same target - # table (e.g. ownerid + createdby + modifiedby all point to systemuser) - # must each get a distinct alias so the combined SQL is valid. - # Expected output: - # ownerid -> systemuser alias=s - # createdby -> systemuser alias=s2 - # modifiedby -> systemuser alias=s3 - log_call("client.query.sql_joins('contact') -- distinct aliases for same target table") - try: - contact_joins = client.query.sql_joins("contact") - systemuser_joins = [j for j in contact_joins if j["target"] == "systemuser"] - print(f"[OK] {len(systemuser_joins)} lookup(s) from contact -> systemuser:") - for j in systemuser_joins: - alias = j["join_clause"].split()[2] - print(f" {j['column']:30s} -> {j['target']} alias={alias}") - aliases = [j["join_clause"].split()[2] for j in contact_joins] - if len(aliases) != len(set(aliases)): - print("[WARN] Duplicate aliases detected") - else: - print(f"[OK] All {len(contact_joins)} aliases unique") - except Exception as e: - print(f"[INFO] Alias check skipped: {e}") - - # sql_join (auto-generate JOIN clause) - log_call(f"client.query.sql_join('{child_table}', '{parent_table}', ...)") - try: - join_clause = client.query.sql_join(child_table, parent_table, from_alias="tk", to_alias="t") - print(f"[OK] {join_clause}") - - sql = f"SELECT TOP 3 tk.new_title, t.new_code FROM {child_table} tk {join_clause}" - results = backoff(lambda: client.query.sql(sql)) - print(f"[OK] Live query with sql_join(): {len(results)} rows") - except Exception as e: - print(f"[WARN] {e}") - # ============================================================== # 30. OData Helper Functions # ============================================================== - heading(30, "OData Helper Functions (query.odata_*)") + heading(30, "OData Helper Functions (query.odata_* — deprecated at GA)") print( - "Parallel helpers for OData/records.get() users -- auto-discover\n" - "navigation properties and build @odata.bind payloads." + "odata_select(), odata_expand(), and odata_bind() still work at GA\n" + "but emit DeprecationWarning. Use the typed query builder instead.\n" + "odata_expands() is kept without deprecation." ) # odata_select @@ -998,7 +956,7 @@ def _run_examples(client): try: nav = client.query.odata_expand(child_table, parent_table) print(f"\n[OK] odata_expand('{child_table}', '{parent_table}') = '{nav}'") - print(" Usage: client.records.get('" + child_table + "', expand=['" + nav + "'])") + print(" Usage: client.query.builder('" + child_table + "').expand('" + nav + "').execute()") except Exception as e: print(f"[WARN] {e}") @@ -1030,8 +988,8 @@ def _run_examples(client): | DISTINCT | YES | Not directly | | Pagination | OFFSET FETCH | @odata.nextLink | | Max results | 5000 per query | 5000 per page | -| Column discovery | sql_columns/sql_select | odata_select | -| JOIN discovery | sql_joins/sql_join | odata_expands/expand | +| Column discovery | sql_columns | odata_expands (kept) | +| JOIN discovery | write manually/fetchxml | odata_expand (deprecated)| | Lookup binding | N/A (read-only) | odata_bind | | SELECT * | YES (SDK auto-expands) | Not applicable | | Polymorphic lookups | Separate JOINs | $expand by nav prop | @@ -1077,21 +1035,20 @@ def _run_examples(client): # OData version (expand) t0 = _time.time() try: - odata_rows = [] - for page in backoff( - lambda: client.records.get( - "account", - select=["name"], - expand=["contact_customer_accounts"], - top=5, + odata_rows = list( + backoff( + lambda: client.records.list( + "account", + select=["name"], + top=5, + ) ) - ): - odata_rows.extend(page) + ) odata_time = _time.time() - t0 - print(f" OData $expand: {len(odata_rows)} rows in {odata_time:.2f}s") + print(f" OData records.list: {len(odata_rows)} rows in {odata_time:.2f}s") except Exception as e: odata_time = _time.time() - t0 - print(f" OData $expand: error ({odata_time:.2f}s): {e}") + print(f" OData records.list: error ({odata_time:.2f}s): {e}") # ============================================================== # 32. Anti-Patterns & Best Practices @@ -1144,8 +1101,8 @@ def _run_examples(client): -> ValidationError (blocked). - Pattern #2 (cartesian FROM a, b) -> UserWarning (advisory). - Server enforces 5000-row cap on all queries (#3, #5). - - Use sql_columns() or sql_select() to discover valid column names. - - Use sql_joins() or sql_join() to discover valid JOIN clauses. + - Use sql_columns() to discover valid column names. + - Write JOIN clauses manually or use fetchxml() for complex queries. """) # ============================================================== @@ -1177,8 +1134,8 @@ def _run_examples(client): | Nested polymorphic chains | YES | e.g. opp -> acct -> contact -> owner | | Audit trail (createdby, etc.) | YES | JOIN to systemuser | | SQL read -> DF write-back | YES | dataframe.sql() + .update()/.create() | -| SQL column discovery | YES | query.sql_columns() / sql_select() | -| SQL JOIN discovery | YES | query.sql_joins() / sql_join() | +| SQL column discovery | YES | query.sql_columns() | +| SQL JOIN clause | manual | write directly or use fetchxml() | | OData column discovery | YES | query.odata_select() | | OData expand discovery | YES | query.odata_expands() / odata_expand() | | OData bind builder | YES | query.odata_bind() | @@ -1195,12 +1152,11 @@ def _run_examples(client): SQL-First Workflow (no OData knowledge needed): 1. Discover schema: cols = client.query.sql_columns("account") - 2. Discover JOINs: joins = client.query.sql_joins("contact") - 3. Build JOIN: j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - 4. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") - 5. Transform: df["col"] = df["col"] * 1.1 - 6. Write back: client.dataframe.update("account", df, id_column="accountid") - 7. Verify: df2 = client.dataframe.sql("SELECT ...") + 2. Write JOIN: j = "JOIN account a ON c.parentcustomerid = a.accountid" + 3. Query with SQL: df = client.dataframe.sql(f"SELECT c.fullname, a.name FROM contact c {j}") + 4. Transform: df["col"] = df["col"] * 1.1 + 5. Write back: client.dataframe.update("account", df, id_column="accountid") + 6. Verify: df2 = client.dataframe.sql("SELECT ...") """) finally: diff --git a/examples/advanced/walkthrough.py b/examples/advanced/walkthrough.py index 6c0a6184..d2cc4ff9 100644 --- a/examples/advanced/walkthrough.py +++ b/examples/advanced/walkthrough.py @@ -26,7 +26,7 @@ from azure.identity import InteractiveBrowserCredential from PowerPlatform.Dataverse.client import DataverseClient from PowerPlatform.Dataverse.core.errors import MetadataError -from PowerPlatform.Dataverse.models.filters import eq, gt, between +from PowerPlatform.Dataverse.models.filters import col from PowerPlatform.Dataverse.models.query_builder import ExpandOption import requests @@ -183,8 +183,8 @@ def _run_walkthrough(client): print("=" * 80) # Single read by ID - log_call(f"client.records.get('{table_name}', '{id1}')") - record = backoff(lambda: client.records.get(table_name, id1)) + log_call(f"client.records.retrieve('{table_name}', '{id1}')") + record = backoff(lambda: client.records.retrieve(table_name, id1)) print("[OK] Retrieved single record:") print( json.dumps( @@ -203,11 +203,8 @@ def _run_walkthrough(client): ) # Multiple read with filter - log_call(f"client.records.get('{table_name}', filter='new_quantity gt 5')") - all_records = [] - records_iterator = backoff(lambda: client.records.get(table_name, filter="new_quantity gt 5")) - for page in records_iterator: - all_records.extend(page) + log_call(f"client.records.list('{table_name}', filter='new_quantity gt 5')") + all_records = list(backoff(lambda: client.records.list(table_name, filter="new_quantity gt 5"))) print(f"[OK] Found {len(all_records)} records with new_quantity > 5") for rec in all_records: print(f" - new_Title='{rec.get('new_title')}', new_Quantity={rec.get('new_quantity')}") @@ -231,7 +228,7 @@ def _run_walkthrough(client): }, ) ) - updated = backoff(lambda: client.records.get(table_name, id1)) + updated = backoff(lambda: client.records.retrieve(table_name, id1)) print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") print(f" new_Notes: {repr(updated.get('new_notes'))}") @@ -263,9 +260,11 @@ def _run_walkthrough(client): print(f"[OK] Created {len(paging_ids)} records for paging demo") # Query with paging - log_call(f"client.records.get('{table_name}', page_size=5)") + log_call(f"client.query.builder('{table_name}').order_by().page_size(5).execute_pages()") print("Fetching records with page_size=5...") - paging_iterator = backoff(lambda: client.records.get(table_name, orderby=["new_Quantity"], page_size=5)) + paging_iterator = backoff( + lambda: client.query.builder(table_name).order_by("new_Quantity").page_size(5).execute_pages() + ) for page_num, page in enumerate(paging_iterator, start=1): record_ids = [r.get("new_walkthroughdemoid")[:8] + "..." for r in page] print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") @@ -278,13 +277,13 @@ def _run_walkthrough(client): print("=" * 80) # Basic fluent query: active records sorted by amount (flat iteration) - log_call("client.query.builder(...).select().filter_eq().order_by().execute()") + log_call("client.query.builder(...).select().where(col(...)==...).order_by().execute()") print("Querying incomplete records ordered by amount (fluent builder)...") qb_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Priority") - .filter_eq("new_Completed", False) + .where(col("new_Completed") == False) .order_by("new_Amount", descending=True) .top(10) .execute() @@ -294,14 +293,14 @@ def _run_walkthrough(client): for rec in qb_records[:5]: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") - # filter_in: records with specific priorities - log_call("client.query.builder(...).filter_in('new_Priority', [HIGH, LOW]).execute()") - print("Querying records with HIGH or LOW priority (filter_in)...") + # col().in_(): records with specific priorities + log_call("client.query.builder(...).where(col('new_Priority').in_([HIGH, LOW])).execute()") + print("Querying records with HIGH or LOW priority (col().in_())...") priority_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Priority") - .filter_in("new_Priority", [Priority.HIGH, Priority.LOW]) + .where(col("new_Priority").in_([Priority.HIGH, Priority.LOW])) .execute() ) ) @@ -309,14 +308,14 @@ def _run_walkthrough(client): for rec in priority_records[:5]: print(f" - '{rec.get('new_title')}' Priority={rec.get('new_priority')}") - # filter_between: amount in a range - log_call("client.query.builder(...).filter_between('new_Amount', 500, 1500).execute()") - print("Querying records with amount between 500 and 1500 (filter_between)...") + # col().between(): amount in a range + log_call("client.query.builder(...).where(col('new_Amount').between(500, 1500)).execute()") + print("Querying records with amount between 500 and 1500 (col().between())...") range_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount") - .filter_between("new_Amount", 500, 1500) + .where(col("new_Amount").between(500, 1500)) .execute() ) ) @@ -325,13 +324,13 @@ def _run_walkthrough(client): print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") # Composable expression tree with where() - log_call("client.query.builder(...).where((eq(...) | eq(...)) & gt(...)).execute()") + log_call("client.query.builder(...).where((col(...) == ...) & (col(...) > ...)).execute()") print("Querying with composable expression tree (where)...") expr_records = list( backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Amount", "new_Quantity") - .where((eq("new_Completed", False) & gt("new_Amount", 100))) + .where((col("new_Completed") == False) & (col("new_Amount") > 100)) .order_by("new_Amount", descending=True) .top(5) .execute() @@ -341,19 +340,19 @@ def _run_walkthrough(client): for rec in expr_records: print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") - # Combined: fluent filters + expression tree + paging (by_page=True) - log_call("client.query.builder(...).filter_eq().where(between()).page_size().execute(by_page=True)") - print("Querying with combined fluent + expression filters and paging...") + # Multiple where() clauses + lazy paging (execute_pages) + log_call("client.query.builder(...).where(col(...)==...).where(col().between()).page_size().execute_pages()") + print("Querying with combined expression filters and paging...") combined_page_count = 0 combined_record_count = 0 for page in backoff( lambda: client.query.builder(table_name) .select("new_Title", "new_Quantity") - .filter_eq("new_Completed", False) - .where(between("new_Quantity", 1, 15)) + .where(col("new_Completed") == False) + .where(col("new_Quantity").between(1, 15)) .order_by("new_Quantity") .page_size(3) - .execute(by_page=True) + .execute_pages() ): combined_page_count += 1 combined_record_count += len(page) @@ -362,13 +361,14 @@ def _run_walkthrough(client): print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") # to_dataframe: get results as a pandas DataFrame - log_call(f"client.query.builder('{table_name}').select(...).filter_eq(...).to_dataframe()") + log_call(f"client.query.builder('{table_name}').select(...).where(col(...)==...).execute().to_dataframe()") print("Querying completed records as a pandas DataFrame (to_dataframe)...") df = backoff( lambda: ( client.query.builder(table_name) .select("new_title", "new_quantity") - .filter_eq("new_completed", True) + .where(col("new_completed") == True) + .execute() .to_dataframe() ) ) @@ -453,7 +453,7 @@ def _run_walkthrough(client): "new_Priority": "High", # String label instead of int } label_id = backoff(lambda: client.records.create(table_name, label_record)) - retrieved = backoff(lambda: client.records.get(table_name, label_id)) + retrieved = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Created record with string label 'High' for new_Priority") print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") @@ -461,7 +461,7 @@ def _run_walkthrough(client): # Update with a string label log_call(f"client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) - updated_label = backoff(lambda: client.records.get(table_name, label_id)) + updated_label = backoff(lambda: client.records.retrieve(table_name, label_id)) print(f"[OK] Updated record with string label 'Low' for new_Priority") print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") print( @@ -615,7 +615,7 @@ def _run_walkthrough(client): print(" [OK] Reading records by ID and with filters") print(" [OK] Single and multiple record updates") print(" [OK] Paging through large result sets") - print(" [OK] QueryBuilder fluent queries (filter_eq, filter_in, filter_between, where, to_dataframe)") + print(" [OK] QueryBuilder fluent queries (where + col(), col().in_(), col().between(), to_dataframe)") print(" [OK] Expand navigation properties (simple + nested ExpandOption)") print(" [OK] SQL queries") print(" [OK] Picklist label-to-value conversion") diff --git a/examples/aio/__init__.py b/examples/aio/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/_auth.py b/examples/aio/_auth.py new file mode 100644 index 00000000..a425321d --- /dev/null +++ b/examples/aio/_auth.py @@ -0,0 +1,57 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async credential helper for the async example scripts. + +azure-identity's InteractiveBrowserCredential is only available in the sync +namespace (azure.identity), not the async one (azure.identity.aio). This +module wraps the sync credential so it satisfies the AsyncTokenCredential +protocol required by AsyncDataverseClient. + +Usage:: + + from _auth import AsyncInteractiveBrowserCredential + + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(org_url, credential) as client: + ... + finally: + await credential.close() +""" + +import asyncio +from concurrent.futures import ThreadPoolExecutor + +from azure.identity import InteractiveBrowserCredential + + +class AsyncInteractiveBrowserCredential: + """ + Async wrapper around the sync InteractiveBrowserCredential. + + get_token() is dispatched to a dedicated thread so the event loop stays + free during the browser popup / token exchange. Subsequent calls hit the + in-process token cache and return almost immediately. + """ + + def __init__(self, **kwargs): + self._credential = InteractiveBrowserCredential(**kwargs) + self._executor = ThreadPoolExecutor(max_workers=1) + + async def get_token(self, *scopes, **kwargs): + loop = asyncio.get_running_loop() + return await loop.run_in_executor( + self._executor, + lambda: self._credential.get_token(*scopes, **kwargs), + ) + + async def close(self): + self._executor.shutdown(wait=False) + + async def __aenter__(self): + return self + + async def __aexit__(self, *_): + await self.close() diff --git a/examples/aio/advanced/__init__.py b/examples/aio/advanced/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/advanced/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/advanced/alternate_keys_upsert.py b/examples/aio/advanced/alternate_keys_upsert.py new file mode 100644 index 00000000..a080975d --- /dev/null +++ b/examples/aio/advanced/alternate_keys_upsert.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Alternate Keys & Upsert Example + +Async equivalent of examples/advanced/alternate_keys_upsert.py. + +Demonstrates the full workflow of creating alternate keys and using +them for upsert operations: +1. Create a custom table with columns +2. Define an alternate key on a column +3. Wait for the key index to become Active +4. Upsert records using the alternate key +5. Verify records were created/updated correctly +6. Clean up + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +# --- Config --- +TABLE_NAME = "new_AltKeyDemo" +KEY_COLUMN = "new_externalid" +KEY_NAME = "new_ExternalIdKey" +BACKOFF_DELAYS = (0, 3, 10, 20, 35) + + +# --- Helpers --- +async def backoff(coro_fn, *, delays=BACKOFF_DELAYS): + """Retry *coro_fn* with exponential-ish backoff on any exception.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" [INFO] Backoff succeeded after {retry_count} retry(s); " f"waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); " f"waited {total_delay}s total.") + raise last + + +async def wait_for_key_active(client, table, key_name, max_wait=120): + """Poll get_alternate_keys until the key status is Active.""" + import time + + start = time.time() + while time.time() - start < max_wait: + keys = await client.tables.get_alternate_keys(table) + for k in keys: + if k.schema_name == key_name: + print(f" Key status: {k.status}") + if k.status == "Active": + return k + if k.status == "Failed": + raise RuntimeError(f"Alternate key index failed: {k.schema_name}") + await asyncio.sleep(5) + raise TimeoutError(f"Key {key_name} did not become Active within {max_wait}s") + + +# --- Main --- +async def main(): + """Run the async alternate-keys & upsert E2E walkthrough.""" + print("PowerPlatform Dataverse Client - Async Alternate Keys & Upsert Example") + print("=" * 70) + print("This script demonstrates:") + print(" - Creating a custom table with columns") + print(" - Defining an alternate key on a column") + print(" - Waiting for the key index to become Active") + print(" - Upserting records via alternate key (create + update)") + print(" - Verifying records and listing keys") + print(" - Cleaning up (delete key, delete table)") + print("=" * 70) + + entered = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not entered: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = entered.rstrip("/") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + + # ------------------------------------------------------------------ + # Step 1: Create table (skip if already exists) + # ------------------------------------------------------------------ + print("\n1. Creating table...") + table_info = await client.tables.get(TABLE_NAME) + if table_info: + print(f" Table already exists: {TABLE_NAME} (skipped)") + else: + table_info = await backoff( + lambda: client.tables.create( + TABLE_NAME, + columns={ + KEY_COLUMN: "string", + "new_ProductName": "string", + "new_Price": "decimal", + }, + ) + ) + print(f" Created: {table_info.get('table_schema_name', TABLE_NAME)}") + await asyncio.sleep(10) # Wait for metadata propagation + + # ------------------------------------------------------------------ + # Step 2: Create alternate key (skip if already exists) + # ------------------------------------------------------------------ + print("\n2. Creating alternate key...") + existing_keys = await client.tables.get_alternate_keys(TABLE_NAME) + existing_key = next((k for k in existing_keys if k.schema_name == KEY_NAME), None) + if existing_key: + print(f" Alternate key already exists: {KEY_NAME} (skipped)") + else: + key_info = await backoff( + lambda: client.tables.create_alternate_key(TABLE_NAME, KEY_NAME, [KEY_COLUMN.lower()]) + ) + print(f" Key created: {key_info.schema_name} (id={key_info.metadata_id})") + + # ------------------------------------------------------------------ + # Step 3: Wait for key to become Active + # ------------------------------------------------------------------ + print("\n3. Waiting for key index to become Active...") + active_key = await wait_for_key_active(client, TABLE_NAME, KEY_NAME) + print(f" Key is Active: {active_key.schema_name}") + + # ------------------------------------------------------------------ + # Step 4: Upsert records (creates new) + # ------------------------------------------------------------------ + print("\n4a. Upsert single record (PATCH, creates new)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-001"}, + record={"new_productname": "Widget A", "new_price": 9.99}, + ), + ], + ) + print(" Upserted EXT-001 (single)") + + print("\n4b. Upsert second record (single PATCH)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-002"}, + record={"new_productname": "Widget B", "new_price": 19.99}, + ), + ], + ) + print(" Upserted EXT-002 (single)") + + print("\n4c. Upsert multiple records (UpsertMultiple bulk)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-003"}, + record={"new_productname": "Widget C", "new_price": 29.99}, + ), + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-004"}, + record={"new_productname": "Widget D", "new_price": 39.99}, + ), + ], + ) + print(" Upserted EXT-003, EXT-004 (bulk)") + + # ------------------------------------------------------------------ + # Step 5a: Upsert single update (PATCH, record exists) + # ------------------------------------------------------------------ + print("\n5a. Upsert single record (update existing via PATCH)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-001"}, + record={"new_productname": "Widget A v2", "new_price": 12.99}, + ), + ], + ) + print(" Updated EXT-001 (single)") + + # ------------------------------------------------------------------ + # Step 5b: Upsert multiple update (UpsertMultiple, records exist) + # ------------------------------------------------------------------ + print("\n5b. Upsert multiple records (update existing via UpsertMultiple)...") + await client.records.upsert( + TABLE_NAME, + [ + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-003"}, + record={"new_productname": "Widget C v2", "new_price": 31.99}, + ), + UpsertItem( + alternate_key={KEY_COLUMN.lower(): "EXT-004"}, + record={"new_productname": "Widget D v2", "new_price": 41.99}, + ), + ], + ) + print(" Updated EXT-003, EXT-004 (bulk)") + + # ------------------------------------------------------------------ + # Step 6: Verify + # ------------------------------------------------------------------ + print("\n6. Verifying records...") + async for record in client.records.list_pages( + TABLE_NAME, + select=["new_productname", "new_price", KEY_COLUMN.lower()], + ): + for item in record: + ext_id = item.get(KEY_COLUMN.lower(), "?") + name = item.get("new_productname", "?") + price = item.get("new_price", "?") + print(f" {ext_id}: {name} @ ${price}") + + # ------------------------------------------------------------------ + # Step 7: List alternate keys + # ------------------------------------------------------------------ + print("\n7. Listing alternate keys...") + keys = await client.tables.get_alternate_keys(TABLE_NAME) + for k in keys: + print(f" {k.schema_name}: columns={k.key_attributes}, status={k.status}") + + # ------------------------------------------------------------------ + # Step 8: Cleanup + # ------------------------------------------------------------------ + cleanup = input("\n8. Delete table and cleanup? (Y/n): ").strip() or "y" + if cleanup.lower() in ("y", "yes"): + try: + # Delete alternate key first + for k in keys: + await client.tables.delete_alternate_key(TABLE_NAME, k.metadata_id) + print(f" Deleted key: {k.schema_name}") + await asyncio.sleep(5) + await backoff(lambda: client.tables.delete(TABLE_NAME)) + print(f" Deleted table: {TABLE_NAME}") + except Exception as e: # noqa: BLE001 + print(f" Cleanup error: {e}") + else: + print(" Table kept for inspection.") + finally: + await credential.close() + + print("\nDone.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/batch.py b/examples/aio/advanced/batch.py new file mode 100644 index 00000000..7023a85b --- /dev/null +++ b/examples/aio/advanced/batch.py @@ -0,0 +1,280 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async batch operations example for the Dataverse Python SDK. + +Async equivalent of examples/advanced/batch.py. + +Demonstrates how to use client.batch to send multiple operations in a single +HTTP request to the Dataverse Web API using the async client. + +Requirements: + pip install PowerPlatform-Dataverse-Client azure-identity +""" + +from __future__ import annotations + +import asyncio +import sys + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + +async def main(): + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + + # --------------------------------------------------------------------------- + # Example 1: Record CRUD in a single batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 1: Record CRUD in a single batch") + + batch = client.batch.new() + + # Create a single record + batch.records.create("account", {"name": "Contoso Ltd", "telephone1": "555-0100"}) + + # Create multiple records via CreateMultiple (one batch item) + batch.records.create( + "contact", + [ + {"firstname": "Alice", "lastname": "Smith"}, + {"firstname": "Bob", "lastname": "Jones"}, + ], + ) + + # Assume we have an existing account_id from a prior operation + # batch.records.update("account", account_id, {"telephone1": "555-9999"}) + # batch.records.delete("account", old_id) + + result = await batch.execute() + + print( + f"[OK] Total: {len(result.responses)}, Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}" + ) + for guid in result.entity_ids: + print(f"[OK] Created: {guid}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 2: Transactional changeset with content-ID chaining + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 2: Transactional changeset") + + batch = client.batch.new() + + async with batch.changeset() as cs: + # Each create() returns a "$n" reference usable in subsequent operations + lead_ref = cs.records.create( + "lead", + {"firstname": "Ada", "lastname": "Lovelace"}, + ) + contact_ref = cs.records.create("contact", {"firstname": "Ada"}) + + # Reference the newly created lead and contact in the account + cs.records.create( + "account", + { + "name": "Babbage & Co.", + "originatingleadid@odata.bind": lead_ref, + "primarycontactid@odata.bind": contact_ref, + }, + ) + + # Update using a content-ID reference as the record_id + cs.records.update("contact", contact_ref, {"lastname": "Lovelace"}) + + result = await batch.execute() + + if result.has_errors: + print("[ERR] Changeset rolled back") + for item in result.failed: + print(f" {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.entity_ids)} records created atomically") + + # --------------------------------------------------------------------------- + # Example 3: Table metadata operations in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 3: Table metadata operations") + + batch = client.batch.new() + + # Create a new custom table + batch.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", + ) + + # Read table metadata + batch.tables.get("new_Product") + + # List all non-private tables + batch.tables.list() + + result = await batch.execute() + print(f"[OK] Table ops: {[(r.status_code, r.is_success) for r in result.responses]}") + + # --------------------------------------------------------------------------- + # Example 4: SQL query in a batch + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 4: SQL query in batch") + + batch = client.batch.new() + batch.query.sql("SELECT TOP 5 accountid, name FROM account ORDER BY name") + + result = await batch.execute() + if result.responses and result.responses[0].is_success and result.responses[0].data: + rows = result.responses[0].data.get("value", []) + print(f"[OK] Retrieved {len(rows)} accounts") + for row in rows: + print(f" {row.get('name')}") + + # --------------------------------------------------------------------------- + # Example 5: Mixed batch — changeset writes + standalone GETs + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 5: Mixed batch") + + # NOTE: Commented out because it requires a pre-existing account_id. + # Uncomment and set account_id to run this example. + # batch = client.batch.new() + # + # async with batch.changeset() as cs: + # cs.records.update("account", account_id, {"statecode": 0}) + # + # batch.records.retrieve("account", account_id, select=["name", "statecode"]) + # + # result = await batch.execute() + # update_response = result.responses[0] + # account_data = result.responses[1] + # if account_data.is_success and account_data.data: + # print(f"Account: {account_data.data.get('name')}") + + # --------------------------------------------------------------------------- + # Example 6: Continue on error + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 6: Continue on error") + + batch = client.batch.new() + batch.records.retrieve("account", "00000000-0000-0000-0000-000000000000") + batch.query.sql("SELECT TOP 1 name FROM account") + + result = await batch.execute(continue_on_error=True) + print(f"[OK] Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") + for item in result.failed: + print(f"[ERR] {item.status_code}: {item.error_message}") + + # --------------------------------------------------------------------------- + # Example 7: DataFrame integration + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 7: DataFrame batch operations") + + import pandas as pd + + # Create records from a DataFrame + df = pd.DataFrame( + [ + {"name": "DF-Batch-A", "telephone1": "555-0100"}, + {"name": "DF-Batch-B", "telephone1": "555-0200"}, + ] + ) + batch = client.batch.new() + batch.dataframe.create("account", df) + result = await batch.execute() + print(f"[OK] DataFrame create: {len(result.succeeded)} succeeded") + created_ids = list(result.entity_ids) + + # Update records from a DataFrame + if len(created_ids) >= 2: + update_df = pd.DataFrame( + [ + {"accountid": created_ids[0], "telephone1": "555-9990"}, + {"accountid": created_ids[1], "telephone1": "555-9991"}, + ] + ) + batch = client.batch.new() + batch.dataframe.update("account", update_df, id_column="accountid") + result = await batch.execute() + print(f"[OK] DataFrame update: {len(result.succeeded)} succeeded") + + # Delete records from a Series + if created_ids: + batch = client.batch.new() + batch.dataframe.delete("account", pd.Series(created_ids), use_bulk_delete=False) + result = await batch.execute() + print(f"[OK] DataFrame delete: {len(result.succeeded)} succeeded") + + # --------------------------------------------------------------------------- + # Example 8: Understanding response data patterns + # --------------------------------------------------------------------------- + + print("\n[INFO] Example 8: Response data patterns") + + # Every batch result maps 1:1 with the operations you added. + # Different operations return different response shapes: + + batch = client.batch.new() + # Op 0: single create -> 204 No Content, entity_id in OData-EntityId header + batch.records.create("account", {"name": "Pattern-Demo"}) + # Op 1: bulk create -> 200 OK, IDs in body as {"Ids": [...]} + batch.records.create("account", [{"name": "Bulk-A"}, {"name": "Bulk-B"}]) + # Op 2: SQL query -> 200 OK, rows in body as {"value": [...]} + batch.query.sql("SELECT TOP 3 name FROM account") + + result = await batch.execute() + + for i, resp in enumerate(result.responses): + if not resp.is_success: + print(f" Op {i}: [FAIL] {resp.status_code}: {resp.error_message}") + continue + + # Single create: entity_id from OData-EntityId header + if resp.entity_id: + print(f" Op {i}: [CREATE] entity_id={resp.entity_id}") + + # Bulk action (CreateMultiple/UpsertMultiple): IDs in body + elif resp.data and "Ids" in resp.data: + print(f" Op {i}: [BULK] {len(resp.data['Ids'])} IDs: {resp.data['Ids']}") + + # Query: rows in body + elif resp.data and "value" in resp.data: + print(f" Op {i}: [QUERY] {len(resp.data['value'])} rows") + + # Delete or metadata operation: 204, no data + else: + print(f" Op {i}: [OK] {resp.status_code}") + + # Clean up demo records + for rid in result.entity_ids: + await client.records.delete("account", rid) + for resp in result.succeeded: + if resp.data and "Ids" in resp.data: + for rid in resp.data["Ids"]: + await client.records.delete("account", rid) + finally: + await credential.close() + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/dataframe_operations.py b/examples/aio/advanced/dataframe_operations.py new file mode 100644 index 00000000..463f38fe --- /dev/null +++ b/examples/aio/advanced/dataframe_operations.py @@ -0,0 +1,191 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async DataFrame Operations Walkthrough + +Async equivalent of examples/advanced/dataframe_operations.py. + +This example demonstrates how to use the async pandas DataFrame extension +methods for CRUD operations with Microsoft Dataverse. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys +import uuid + +import pandas as pd +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw + + +async def main(): + # -- Setup & Authentication ------------------------------------ + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + await _run_walkthrough(client) + finally: + await credential.close() + + +async def _run_walkthrough(client): + table = input("Enter table schema name to use [default: account]: ").strip() or "account" + print(f"[INFO] Using table: {table}") + + # Unique tag to isolate test records from existing data + tag = uuid.uuid4().hex[:8] + test_filter = f"contains(name,'{tag}')" + print(f"[INFO] Using tag '{tag}' to identify test records") + + select_cols = ["name", "telephone1", "websiteurl", "lastonholdtime"] + + # -- 1. Create records from a DataFrame ------------------------ + print("\n" + "-" * 60) + print("1. Create records from a DataFrame") + print("-" * 60) + + new_accounts = pd.DataFrame( + [ + { + "name": f"Contoso_{tag}", + "telephone1": "555-0100", + "websiteurl": "https://contoso.com", + "lastonholdtime": pd.Timestamp("2024-06-15 10:30:00"), + }, + {"name": f"Fabrikam_{tag}", "telephone1": "555-0200", "websiteurl": None, "lastonholdtime": None}, + { + "name": f"Northwind_{tag}", + "telephone1": None, + "websiteurl": "https://northwind.com", + "lastonholdtime": pd.Timestamp("2024-12-01 08:00:00"), + }, + ] + ) + print(f" Input DataFrame:\n{new_accounts.to_string(index=False)}\n") + + # create returns a Series of GUIDs aligned with the input rows + new_accounts["accountid"] = await client.dataframe.create(table, new_accounts) + print(f"[OK] Created {len(new_accounts)} records") + print(f" IDs: {new_accounts['accountid'].tolist()}") + + # -- 2. Query records as a DataFrame ------------------------- + print("\n" + "-" * 60) + print("2. Query records as a DataFrame") + print("-" * 60) + + result = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + df_all = result.to_dataframe() + print(f"[OK] Got {len(df_all)} records in one DataFrame") + print(f" Columns: {list(df_all.columns)}") + print(f"{df_all.to_string(index=False)}") + + # -- 3. Limit results with top ------------------------------ + print("\n" + "-" * 60) + print("3. Limit results with top") + print("-" * 60) + + result_top2 = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).top(2).execute() + df_top2 = result_top2.to_dataframe() + print(f"[OK] Got {len(df_top2)} records with top=2") + print(f"{df_top2.to_string(index=False)}") + + # -- 4. Fetch a single record by ID ---------------------------- + print("\n" + "-" * 60) + print("4. Fetch a single record by ID") + print("-" * 60) + + first_id = new_accounts["accountid"].iloc[0] + print(f" Fetching record {first_id}...") + result_single = await client.query.builder(table).select(*select_cols).where(col("accountid") == first_id).execute() + single = result_single.to_dataframe() + print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}") + + # -- 5. Update records with different values per row ----------- + print("\n" + "-" * 60) + print("5. Update records with different values per row") + print("-" * 60) + + new_accounts["telephone1"] = ["555-1100", "555-1200", "555-1300"] + print(f" New telephone numbers: {new_accounts['telephone1'].tolist()}") + await client.dataframe.update(table, new_accounts[["accountid", "telephone1"]], id_column="accountid") + print("[OK] Updated 3 records") + + # Verify the updates + result_verified = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + verified = result_verified.to_dataframe() + print(f" Verified:\n{verified.to_string(index=False)}") + + # -- 6. Broadcast update (same value to all records) ----------- + print("\n" + "-" * 60) + print("6. Broadcast update (same value to all records)") + print("-" * 60) + + broadcast_df = new_accounts[["accountid"]].copy() + broadcast_df["websiteurl"] = "https://updated.example.com" + print(f" Setting websiteurl to 'https://updated.example.com' for all {len(broadcast_df)} records") + await client.dataframe.update(table, broadcast_df, id_column="accountid") + print("[OK] Broadcast update complete") + + # Verify all records have the same websiteurl + result_bc = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print(f" Verified:\n{result_bc.to_dataframe().to_string(index=False)}") + + # Default: NaN/None fields are skipped (not overridden on server) + print("\n Updating with NaN values (default: clear_nulls=False, fields should stay unchanged)...") + sparse_df = pd.DataFrame( + [ + {"accountid": new_accounts["accountid"].iloc[0], "telephone1": "555-9999", "websiteurl": None}, + ] + ) + await client.dataframe.update(table, sparse_df, id_column="accountid") + result_sparse = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print( + f" Verified (Contoso telephone1 updated, websiteurl unchanged):\n" + f"{result_sparse.to_dataframe().to_string(index=False)}" + ) + + # Opt-in: clear_nulls=True sends None as null to clear the field + print("\n Clearing websiteurl for Contoso with clear_nulls=True...") + clear_df = pd.DataFrame([{"accountid": new_accounts["accountid"].iloc[0], "websiteurl": None}]) + await client.dataframe.update(table, clear_df, id_column="accountid", clear_nulls=True) + result_clear = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + print(f" Verified (Contoso websiteurl should be empty):\n" f"{result_clear.to_dataframe().to_string(index=False)}") + + # -- 7. Delete records by passing a Series of GUIDs ------------ + print("\n" + "-" * 60) + print("7. Delete records by passing a Series of GUIDs") + print("-" * 60) + + print(f" Deleting {len(new_accounts)} records...") + await client.dataframe.delete(table, new_accounts["accountid"], use_bulk_delete=False) + print(f"[OK] Deleted {len(new_accounts)} records") + + # Verify deletions -- filter for our tagged records should return 0 + result_remaining = await client.query.builder(table).select(*select_cols).where(raw(test_filter)).execute() + remaining = result_remaining.to_dataframe() + print(f" Verified: {len(remaining)} test records remaining (expected 0)") + + print("\n" + "=" * 60) + print("[OK] Async DataFrame operations walkthrough complete!") + print("=" * 60) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/datascience_risk_assessment.py b/examples/aio/advanced/datascience_risk_assessment.py new file mode 100644 index 00000000..8d88fd68 --- /dev/null +++ b/examples/aio/advanced/datascience_risk_assessment.py @@ -0,0 +1,665 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Data Science Risk Assessment Pipeline + +Async equivalent of examples/advanced/datascience_risk_assessment.py. + +End-to-end example: Extract Dataverse data concurrently into DataFrames, +run statistical analysis, generate LLM-powered risk summaries, and write +results back to Dataverse -- a realistic data analyst / data scientist workflow. + +Pipeline flow: + Dataverse SDK (async) --> Pandas DataFrame --> Analysis + LLM --> Write-back & Reports + +The three Dataverse extraction queries (accounts, cases, opportunities) run +concurrently via asyncio.gather(), reducing wall-clock time for the extract step. + +Scenario: + A financial services company tracks customer accounts, service cases, and + revenue opportunities in Dataverse. The risk team needs to: + 1) Pull data from multiple tables into DataFrames (concurrently) + 2) Compute risk scores using statistical analysis (pandas/numpy) + 3) Classify and summarize risk using an LLM + 4) Write risk assessments back to Dataverse + 5) Produce a summary report + + Note: This example reads from existing Dataverse tables (account, + incident, opportunity) and does not create or delete any tables. + Step 4 (write-back) is disabled by default -- uncomment it in + run_risk_pipeline() to write risk scores back to account records. + +Prerequisites (required -- included in SDK dependencies): + pip install PowerPlatform-Dataverse-Client + pip install azure-identity + +Additional libraries (optional -- used for visualization and LLM): + pip install matplotlib + pip install azure-ai-inference # Option A: Azure AI Foundry / Azure OpenAI + pip install openai # Option B: OpenAI / Azure OpenAI +""" + +import asyncio +import sys +import warnings +from pathlib import Path +from textwrap import dedent + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import numpy as np +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col, raw + +# -- Optional imports (graceful degradation if not installed) ------ + +try: + import matplotlib + + matplotlib.use("Agg") # non-interactive backend (no GUI required) + import matplotlib.pyplot as plt + + HAS_MATPLOTLIB = True +except ImportError: + HAS_MATPLOTLIB = False + + +# ================================================================ +# LLM Provider Configuration +# ================================================================ +# Same providers as the sync version. LLM calls are kept synchronous +# here since they are CPU-light blocking calls. Replace with async +# LLM clients (e.g. openai.AsyncOpenAI) if latency matters. + + +def get_llm_client(provider=None, endpoint=None, api_key=None, model="gpt-4o"): + """Create an LLM client using the specified (or first available) provider. + + Returns a callable: llm_complete(system_prompt, user_prompt) -> str + Returns None if no provider is available. + """ + providers = [provider] if provider else ["azure-ai-inference", "openai", "copilot-sdk"] + for p in providers: + client = _try_init_provider(p, endpoint, api_key, model) + if client is not None: + return client + return None + + +def _wrap_with_logging(raw_complete, provider_name, model_name): + import time + + log = [] + + def complete(system_prompt, user_prompt): + start = time.time() + response = raw_complete(system_prompt, user_prompt) + elapsed = time.time() - start + log.append( + { + "provider": provider_name, + "model": model_name, + "system_prompt": system_prompt, + "user_prompt": user_prompt, + "response": response, + "elapsed_seconds": round(elapsed, 2), + } + ) + return response + + complete.log = log + complete.provider_name = provider_name + complete.model_name = model_name + return complete + + +def _try_init_provider(name, endpoint, api_key, model): + if name == "azure-ai-inference": + return _init_azure_ai(endpoint, api_key, model) + elif name == "openai": + return _init_openai(endpoint, api_key, model) + elif name == "copilot-sdk": + return _init_copilot_sdk() + return None + + +def _init_azure_ai(endpoint, api_key, model): + try: + from azure.ai.inference import ChatCompletionsClient + from azure.ai.inference.models import SystemMessage, UserMessage + from azure.core.credentials import AzureKeyCredential + except ImportError: + return None + + if not endpoint or not api_key: + return None + + client = ChatCompletionsClient(endpoint=endpoint, credential=AzureKeyCredential(api_key)) + + def complete(system_prompt, user_prompt): + response = client.complete( + messages=[SystemMessage(content=system_prompt), UserMessage(content=user_prompt)], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + print("[INFO] LLM provider: Azure AI Inference") + return _wrap_with_logging(complete, "Azure AI Inference", model) + + +def _init_openai(endpoint, api_key, model): + try: + import openai + except ImportError: + return None + + if not api_key: + return None + + if endpoint: + client = openai.AzureOpenAI(azure_endpoint=endpoint, api_key=api_key, api_version="2024-02-01") + else: + client = openai.OpenAI(api_key=api_key) + + def complete(system_prompt, user_prompt): + response = client.chat.completions.create( + model=model, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_prompt}, + ], + max_tokens=150, + temperature=0.3, + ) + return response.choices[0].message.content.strip() + + provider_name = "Azure OpenAI" if endpoint else "OpenAI" + print(f"[INFO] LLM provider: {provider_name}") + return _wrap_with_logging(complete, provider_name, model) + + +def _init_copilot_sdk(): + # Uncomment and configure to use your Copilot subscription as the LLM provider. + # from copilot import CopilotClient + # ... + return None + + +# ================================================================ +# Configuration +# ================================================================ + +TABLE_ACCOUNTS = "account" +TABLE_CASES = "incident" +TABLE_OPPORTUNITIES = "opportunity" + +RISK_HIGH = 75 +RISK_MEDIUM = 40 + +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "risk_assessment_output" + + +async def main(): + """Entry point -- authenticate and run the async pipeline.""" + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + await run_risk_pipeline(client) + finally: + await credential.close() + + +# ================================================================ +# Step 1: Extract -- Pull data concurrently with asyncio.gather +# ================================================================ + + +async def step1_extract(client): + """Extract accounts, cases, and opportunities concurrently.""" + print("\n" + "=" * 60) + print("STEP 1: Extract data from Dataverse (concurrently)") + print("=" * 60) + + # All three queries run in parallel -- significant speedup vs sequential. + accounts_result, cases_result, opps_result = await asyncio.gather( + client.query.builder(TABLE_ACCOUNTS) + .select("accountid", "name", "revenue", "numberofemployees", "industrycode") + .where(col("statecode") == 0) + .top(200) + .execute(), + client.query.builder(TABLE_CASES) + .select("incidentid", "_customerid_value", "title", "severitycode", "prioritycode", "createdon") + .where(raw("statecode eq 0")) + .top(1000) + .execute(), + client.query.builder(TABLE_OPPORTUNITIES) + .select( + "opportunityid", + "_parentaccountid_value", + "name", + "estimatedvalue", + "closeprobability", + "estimatedclosedate", + ) + .where(col("statecode") == 0) + .top(1000) + .execute(), + ) + + accounts = accounts_result.to_dataframe() + cases = cases_result.to_dataframe() + opportunities = opps_result.to_dataframe() + + print(f"[OK] Extracted {len(accounts)} active accounts") + print(f"[OK] Extracted {len(cases)} open cases") + print(f"[OK] Extracted {len(opportunities)} active opportunities") + + return accounts, cases, opportunities + + +# ================================================================ +# Step 2: Transform & Analyze -- Statistical risk scoring +# ================================================================ + + +def step2_analyze(accounts, cases, opportunities): + """Compute risk scores using pandas statistical operations (pure Python, unchanged).""" + print("\n" + "=" * 60) + print("STEP 2: Statistical analysis -- compute risk scores") + print("=" * 60) + + if not cases.empty and "_customerid_value" in cases.columns: + case_stats = ( + cases.groupby("_customerid_value") + .agg( + total_cases=("incidentid", "count"), + high_severity_cases=("severitycode", lambda x: (x == 1).sum()), + avg_priority=("prioritycode", "mean"), + ) + .reset_index() + .rename(columns={"_customerid_value": "accountid"}) + ) + else: + case_stats = pd.DataFrame(columns=["accountid", "total_cases", "high_severity_cases", "avg_priority"]) + + if not opportunities.empty and "_parentaccountid_value" in opportunities.columns: + opportunities = opportunities.copy() + opportunities["_weighted_value"] = ( + pd.to_numeric(opportunities["estimatedvalue"], errors="coerce").fillna(0) + * pd.to_numeric(opportunities["closeprobability"], errors="coerce").fillna(0) + / 100 + ) + opp_stats = ( + opportunities.groupby("_parentaccountid_value") + .agg( + total_opportunities=("opportunityid", "count"), + pipeline_value=("estimatedvalue", "sum"), + avg_close_probability=("closeprobability", "mean"), + weighted_pipeline=("_weighted_value", "sum"), + ) + .reset_index() + .rename(columns={"_parentaccountid_value": "accountid"}) + ) + else: + opp_stats = pd.DataFrame( + columns=[ + "accountid", + "total_opportunities", + "pipeline_value", + "avg_close_probability", + "weighted_pipeline", + ] + ) + + risk_df = accounts.merge(case_stats, on="accountid", how="left") + risk_df = risk_df.merge(opp_stats, on="accountid", how="left") + + for c in ["revenue", "numberofemployees"]: + if c in risk_df.columns: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0) + + for c in ["total_cases", "high_severity_cases"]: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0).astype(int) + for c in ["avg_priority", "pipeline_value", "avg_close_probability", "weighted_pipeline"]: + risk_df[c] = pd.to_numeric(risk_df[c], errors="coerce").fillna(0).astype(float) + risk_df["total_opportunities"] = ( + pd.to_numeric(risk_df["total_opportunities"], errors="coerce").fillna(0).astype(int) + ) + + risk_df["risk_score"] = compute_risk_score(risk_df) + risk_df["risk_tier"] = risk_df["risk_score"].apply(classify_risk) + + print(f"[OK] Computed risk scores for {len(risk_df)} accounts") + print(f" High risk: {(risk_df['risk_tier'] == 'High').sum()}") + print(f" Medium risk: {(risk_df['risk_tier'] == 'Medium').sum()}") + print(f" Low risk: {(risk_df['risk_tier'] == 'Low').sum()}") + + print("\n Risk score distribution:") + print(f" Mean: {risk_df['risk_score'].mean():.1f}") + print(f" Median: {risk_df['risk_score'].median():.1f}") + print(f" Std: {risk_df['risk_score'].std():.1f}") + print(f" Min: {risk_df['risk_score'].min():.1f}") + print(f" Max: {risk_df['risk_score'].max():.1f}") + + return risk_df + + +def compute_risk_score(df): + """Compute a 0-100 risk score from multiple factors.""" + scores = pd.Series(0.0, index=df.index) + + case_total = df["total_cases"].clip(lower=1) + severity_ratio = df["high_severity_cases"] / case_total + scores += severity_ratio * 35 + + if df["total_cases"].max() > 0: + case_pctile = df["total_cases"].rank(pct=True) + scores += case_pctile * 25 + else: + scores += 12.5 + + max_pipeline = df["weighted_pipeline"].max() + if max_pipeline > 0: + pipeline_strength = df["weighted_pipeline"] / max_pipeline + scores += (1 - pipeline_strength) * 20 + else: + scores += 10 + + close_risk = (100 - df["avg_close_probability"]) / 100 + scores += close_risk * 20 + + return scores.clip(0, 100).round(1) + + +def classify_risk(score): + if score >= RISK_HIGH: + return "High" + elif score >= RISK_MEDIUM: + return "Medium" + return "Low" + + +# ================================================================ +# Step 3: LLM Summarization +# ================================================================ + + +def step3_summarize(risk_df, llm_complete=None): + """Generate per-account risk summaries using LLM or template fallback.""" + print("\n" + "=" * 60) + print("STEP 3: Generate risk summaries") + print("=" * 60) + + flagged = risk_df[risk_df["risk_tier"].isin(["High", "Medium"])].copy() + print(f"[INFO] Generating summaries for {len(flagged)} flagged accounts") + + if llm_complete is not None: + summaries = _summarize_with_llm(flagged, llm_complete) + if hasattr(llm_complete, "log") and llm_complete.log: + _export_llm_log(llm_complete) + else: + print("[INFO] No LLM provider configured -- using template-based summarization") + summaries = _summarize_with_template(flagged) + + flagged["risk_summary"] = summaries + summary_map = dict(zip(flagged["accountid"], flagged["risk_summary"])) + risk_df["risk_summary"] = risk_df["accountid"].map(summary_map).fillna("Low risk -- no action needed.") + + print(f"[OK] Generated {len(summaries)} risk summaries") + + top_risk = risk_df.nlargest(3, "risk_score") + for _, row in top_risk.iterrows(): + print(f"\n Account: {row.get('name', 'Unknown')}") + print(f" Risk Score: {row['risk_score']} ({row['risk_tier']})") + print(f" Summary: {row['risk_summary'][:120]}...") + + return risk_df + + +def _summarize_with_llm(flagged_df, llm_complete): + system_prompt = ( + "You are a customer risk analyst at a financial services company. " + "Write exactly 2-3 sentences per account. " + "Sentence 1: State the risk level and primary driver. " + "Sentence 2: Quantify the key metric(s) behind the risk. " + "Sentence 3 (if needed): Recommend one specific action. " + "Use plain business language. Do not use bullet points or markdown." + ) + + summaries = [] + for _, row in flagged_df.iterrows(): + user_prompt = dedent(f"""\ + Summarize the risk for this account: + + Account Name: {row.get("name", "Unknown")} + Risk Score: {row["risk_score"]:.0f}/100 ({row["risk_tier"]} risk) + Open Support Cases: {row["total_cases"]} total, {row["high_severity_cases"]} high-severity + Revenue Pipeline: ${row["pipeline_value"]:,.0f} total, ${row["weighted_pipeline"]:,.0f} probability-weighted + Average Deal Close Probability: {row["avg_close_probability"]:.0f}% + """) + summaries.append(llm_complete(system_prompt, user_prompt)) + + return summaries + + +def _summarize_with_template(flagged_df): + summaries = [] + for _, row in flagged_df.iterrows(): + name = row.get("name", "Unknown") + parts = [] + + if row["high_severity_cases"] > 0: + parts.append(f"{row['high_severity_cases']} high-severity cases require immediate attention") + if row["total_cases"] > 5: + parts.append(f"elevated case volume ({row['total_cases']} open)") + if row["weighted_pipeline"] < 10000: + parts.append("weak revenue pipeline") + if row["avg_close_probability"] < 30: + parts.append(f"low close probability ({row['avg_close_probability']:.0f}%)") + if not parts: + parts.append("multiple moderate risk factors detected") + + summary = ( + f"{name} has a {row['risk_tier'].lower()} risk score of " + f"{row['risk_score']:.0f}/100. Key factors: {'; '.join(parts)}. " + f"Recommend proactive outreach and account review." + ) + summaries.append(summary) + + return summaries + + +def _export_llm_log(llm_complete, include_prompts=False): + log_path = OUTPUT_DIR / "llm_interactions.txt" + with open(log_path, "w", encoding="utf-8") as f: + f.write("LLM Interaction Log\n") + f.write("=" * 70 + "\n") + f.write(f"Provider: {llm_complete.provider_name}\n") + f.write(f"Model: {llm_complete.model_name}\n") + f.write(f"Total calls: {len(llm_complete.log)}\n") + total_time = sum(entry["elapsed_seconds"] for entry in llm_complete.log) + f.write(f"Total time: {total_time:.1f}s\n") + f.write("=" * 70 + "\n\n") + + for i, entry in enumerate(llm_complete.log, 1): + f.write(f"--- Call {i} ({entry['elapsed_seconds']:.2f}s) ---\n\n") + if include_prompts: + f.write(f"[System Prompt]\n{entry['system_prompt']}\n\n") + f.write(f"[User Prompt]\n{entry['user_prompt']}\n\n") + f.write(f"[Response]\n{entry['response']}\n\n") + else: + f.write(f"[Response length: {len(entry['response'])} chars]\n\n") + + print(f"[OK] LLM interaction log saved to {log_path}") + + +# ================================================================ +# Step 4: Write-back +# ================================================================ + + +async def step4_writeback(client, risk_df): + """Write risk scores and summaries back to Dataverse accounts.""" + print("\n" + "=" * 60) + print("STEP 4: Write risk assessments back to Dataverse") + print("=" * 60) + + update_df = risk_df[["accountid", "description"]].copy() + update_df["description"] = risk_df.apply( + lambda r: f"[Risk: {r['risk_tier']} ({r['risk_score']:.0f}/100)] {r['risk_summary']}", + axis=1, + ) + + await client.dataframe.update(TABLE_ACCOUNTS, update_df, id_column="accountid") + print(f"[OK] Updated {len(update_df)} account records with risk assessments") + + +# ================================================================ +# Step 5: Report +# ================================================================ + + +def step5_report(risk_df): + """Generate a summary report with optional visualization.""" + print("\n" + "=" * 60) + print("STEP 5: Risk assessment report") + print("=" * 60) + + tier_summary = ( + risk_df.groupby("risk_tier") + .agg( + count=("accountid", "count"), + avg_score=("risk_score", "mean"), + total_cases=("total_cases", "sum"), + total_pipeline=("pipeline_value", "sum"), + ) + .round(1) + ) + print("\nRisk Tier Summary:") + print(tier_summary.to_string()) + + top10 = risk_df.nlargest(10, "risk_score")[ + ["name", "risk_score", "risk_tier", "total_cases", "high_severity_cases", "pipeline_value"] + ] + print("\nTop 10 Highest Risk Accounts:") + print(top10.to_string(index=False)) + + if HAS_MATPLOTLIB: + _generate_charts(risk_df) + else: + print("\n[INFO] Install matplotlib for risk visualization charts") + + risk_df.to_csv(OUTPUT_DIR / "risk_scores.csv", index=False) + top10.to_csv(OUTPUT_DIR / "top10_risk.csv", index=False) + tier_summary.to_csv(OUTPUT_DIR / "tier_summary.csv") + print(f"\n[OK] Exported CSV reports to {OUTPUT_DIR}/") + + print("\n[OK] Risk assessment pipeline complete!") + + +def _generate_charts(risk_df): + fig, axes = plt.subplots(1, 3, figsize=(16, 5)) + fig.suptitle("Customer Account Risk Assessment", fontsize=14, fontweight="bold") + + axes[0].hist(risk_df["risk_score"], bins=20, color="#4472C4", edgecolor="white") + axes[0].axvline(RISK_HIGH, color="red", linestyle="--", label=f"High ({RISK_HIGH})") + axes[0].axvline(RISK_MEDIUM, color="orange", linestyle="--", label=f"Medium ({RISK_MEDIUM})") + axes[0].set_title("Risk Score Distribution") + axes[0].set_xlabel("Risk Score") + axes[0].set_ylabel("Number of Accounts") + axes[0].legend() + + tier_counts = risk_df["risk_tier"].value_counts() + colors = {"High": "#FF4444", "Medium": "#FFA500", "Low": "#44BB44"} + axes[1].pie( + tier_counts.values, + labels=tier_counts.index, + colors=[colors.get(t, "#888") for t in tier_counts.index], + autopct="%1.0f%%", + startangle=90, + ) + axes[1].set_title("Risk Tier Breakdown") + + axes[2].scatter( + risk_df["total_cases"], + risk_df["pipeline_value"], + c=risk_df["risk_score"], + cmap="RdYlGn_r", + alpha=0.7, + edgecolors="gray", + s=60, + ) + axes[2].set_title("Cases vs Pipeline (color = risk)") + axes[2].set_xlabel("Open Cases") + axes[2].set_ylabel("Pipeline Value ($)") + + plt.tight_layout() + chart_path = OUTPUT_DIR / "risk_assessment_report.png" + plt.savefig(chart_path, dpi=150, bbox_inches="tight") + print(f"[OK] Saved {chart_path}") + + +# ================================================================ +# Pipeline Orchestrator +# ================================================================ + + +async def run_risk_pipeline(client): + """Run the full async risk assessment pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + print("\n" + "#" * 60) + print(" ASYNC CUSTOMER RISK ASSESSMENT PIPELINE") + print(" Dataverse SDK (async) -> Pandas -> Analysis -> LLM -> Write-back") + print("#" * 60) + + # Step 1: Extract data concurrently + accounts, cases, opportunities = await step1_extract(client) + + if accounts.empty: + print("[WARN] No accounts found -- nothing to analyze.") + return + + # Step 2: Statistical analysis (pure Python -- synchronous) + risk_df = step2_analyze(accounts, cases, opportunities) + + # Step 3: LLM-powered risk summarization (synchronous LLM calls) + # Configure your LLM provider (uncomment one): + # Option A: Azure AI Inference + # llm = get_llm_client("azure-ai-inference", endpoint="https://...", api_key="...") + # Option B: OpenAI + # llm = get_llm_client("openai", api_key="sk-...") + # Option C: Azure OpenAI (via openai package) + # llm = get_llm_client("openai", endpoint="https://...", api_key="...") + llm = None # Set to get_llm_client(...) to enable LLM summarization + risk_df = step3_summarize(risk_df, llm_complete=llm) + + # Step 4: Write results back to Dataverse (async) + # Uncomment the next line to write back (requires custom columns on account table) + # await step4_writeback(client, risk_df) + print("\n[INFO] Step 4 (write-back) is commented out by default.") + print(" Uncomment step4_writeback() after adding custom columns to account table.") + + # Step 5: Generate summary report + charts (synchronous) + step5_report(risk_df) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/fetchxml.py b/examples/aio/advanced/fetchxml.py new file mode 100644 index 00000000..95fd8811 --- /dev/null +++ b/examples/aio/advanced/fetchxml.py @@ -0,0 +1,574 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async end-to-end FetchXML examples for Dataverse. + +Async equivalent of examples/advanced/fetchxml.py. + +Demonstrates ``await client.query.fetchxml().execute()`` and +``async for page in client.query.fetchxml().execute_pages()`` across +scenarios where FetchXML is required or preferred over OData/SQL: + +- Basic attribute queries +- operators (eq, like, in, null, not-null, between) +- (inner and outer joins) +- Ordering +- Page-size control with automatic paging-cookie propagation +- Aggregate queries (count, sum, avg, min, max, group-by) +- Built-in system tables (account → contact join) + +FetchXML is the right tool when: +- You need a JOIN type OData $expand cannot express (many-to-many, outer link) +- You need server-side aggregates (count, sum, avg) without GROUP BY SQL +- You need ```` operators unavailable in OData ($filter) + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import asyncio +import sys + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK -- Async FetchXML End-to-End Examples") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_examples(client) + finally: + await credential.close() + + +async def _run_examples(client): + project_table = "new_FXDemoProject" + task_table = "new_FXDemoTask" + + # =================================================================== + # 2. Create tables and seed data + # =================================================================== + heading(2, "Create Tables & Seed Data") + + log_call(f"await client.tables.get('{project_table}')") + if await client.tables.get(project_table): + print(f"[OK] Table already exists: {project_table}") + else: + log_call(f"await client.tables.create('{project_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + project_table, + { + "new_Code": "string", + "new_Budget": "decimal", + "new_Active": "bool", + "new_Region": "int", + }, + ) + ) + print(f"[OK] Created table: {project_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {project_table} (skipped)") + else: + raise + + log_call(f"await client.tables.get('{task_table}')") + if await client.tables.get(task_table): + print(f"[OK] Table already exists: {task_table}") + else: + log_call(f"await client.tables.create('{task_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + task_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {task_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {task_table} (skipped)") + else: + raise + + print("\n[INFO] Creating lookup field: tasks → projects ...") + try: + await client.tables.create_lookup_field( + referencing_table=task_table, + lookup_field_name="new_ProjectId", + referenced_table=project_table, + display_name="Project", + ) + print("[OK] Created lookup: new_ProjectId on tasks → projects") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + # Resolve entity set name for @odata.bind + project_set = f"{project_table.lower()}s" + try: + tinfo = await client.tables.get(project_table) + if tinfo: + project_set = tinfo.get("entity_set_name", project_set) + except Exception: + pass + + log_call(f"await client.records.create('{project_table}', [...])") + projects = [ + {"new_Code": "ALPHA", "new_Budget": 50000, "new_Active": True, "new_Region": 1}, + {"new_Code": "BRAVO", "new_Budget": 75000, "new_Active": True, "new_Region": 2}, + {"new_Code": "CHARLIE", "new_Budget": 30000, "new_Active": False, "new_Region": 3}, + {"new_Code": "DELTA", "new_Budget": 90000, "new_Active": True, "new_Region": 1}, + {"new_Code": "ECHO", "new_Budget": 42000, "new_Active": True, "new_Region": 2}, + ] + project_ids = await backoff(lambda: client.records.create(project_table, projects)) + print(f"[OK] Seeded {len(project_ids)} projects") + + log_call(f"await client.records.create('{task_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + "new_ProjectId@odata.bind": f"/{project_set}({project_ids[4]})", + }, + ] + task_ids = await backoff(lambda: client.records.create(task_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks") + + project_logical = project_table.lower() + task_logical = task_table.lower() + project_pk = f"{project_logical}id" + lookup_attr = "new_projectid" + + try: + # =================================================================== + # 3. Basic attribute query + # =================================================================== + heading(3, "Basic Attribute Query") + xml = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(basic attribute query).execute()") + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} projects:") + for r in result: + print(f" {r.get('new_code', ''):<10s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + if result: + print(f" First by index: {result[0].get('new_code')}") + print(f" First by .first(): {result.first().get('new_code')}") + + # =================================================================== + # 4. operators + # =================================================================== + heading(4, " Operators") + + # eq + xml = f""" + + + + + + + """ + log_call('operator="eq" value="ALPHA"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] eq: {[x.get('new_code') for x in r]}") + + # like + xml = f""" + + + + + + + """ + log_call('operator="like" value="%test%"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] like: {len(r)} matches -> {[x.get('new_title') for x in r]}") + + # in + xml = f""" + + + + + + ALPHA + DELTA + + + + + """ + log_call('operator="in" values=[ALPHA, DELTA]') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] in: {[x.get('new_code') for x in r]}") + + # not-null + xml = f""" + + + + + + + """ + log_call('operator="not-null"') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] not-null: {len(r)} tasks have priority set") + + # between + xml = f""" + + + + + + + 40000 + 80000 + + + + + """ + log_call('operator="between" 40000 and 80000') + r = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] between: {len(r)} projects -> {[(x.get('new_code'), x.get('new_budget')) for x in r]}") + + # =================================================================== + # 5. — inner join + # =================================================================== + heading(5, " Inner Join (Tasks → Projects)") + xml = f""" + + + + + + + + + + + """ + log_call("await client.query.fetchxml(link-entity inner join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows:") + for r in result: + print( + f" Task={r.get('new_title', ''):<25s} " + f"Hours={r.get('new_hours')} " + f"Project={r.get('p.new_code', '')} " + f"Budget={r.get('p.new_budget')}" + ) + except Exception as e: + print(f"[WARN] link-entity join failed: {e}") + + # =================================================================== + # 6. — outer join + # =================================================================== + heading(6, " Outer Join (Projects With or Without Tasks)") + xml = f""" + + + + + + + + + """ + log_call("await client.query.fetchxml(link-entity outer join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} rows (includes projects with no tasks):") + for r in result[:8]: + print(f" Project={r.get('new_code', ''):<10s} Task={r.get('t.new_title', '(none)')}") + except Exception as e: + print(f"[WARN] outer join failed: {e}") + + # =================================================================== + # 7. Ordering + # =================================================================== + heading(7, "Ordering ( element)") + xml = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(order by hours DESC).execute()") + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Tasks by hours DESC:") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + # =================================================================== + # 8. Paging-cookie propagation + # =================================================================== + heading(8, "Paging-Cookie Propagation") + print( + "[INFO] 'count' sets the page size in FetchXML.\n" + "With count='2' and 7 seeded tasks the server returns pages of 2, 2, 2, 1.\n" + ".execute() collects all pages eagerly; .execute_pages() yields one QueryResult per HTTP page." + ) + xml_paged = f""" + + + + + + + + """ + log_call("await client.query.fetchxml(xml).execute() — eager, all pages collected") + result = await backoff(lambda: client.query.fetchxml(xml_paged).execute()) + print(f"[OK] execute(): {len(result)} total tasks (seeded {len(task_ids)}):") + for r in result: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + + log_call("async for page in client.query.fetchxml(xml).execute_pages() — lazy, one QueryResult per page") + page_num = 0 + page_record_count = 0 + async for page in client.query.fetchxml(xml_paged).execute_pages(): + page_num += 1 + page_record_count += len(page) + print(f" Page {page_num}: {len(page)} record(s) — {[r.get('new_title') for r in page]}") + print(f"[OK] execute_pages(): {page_record_count} total tasks across {page_num} page(s)") + + # =================================================================== + # 9. Aggregates + # =================================================================== + heading(9, "Aggregate Queries ()") + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(aggregate: count, sum, avg, min, max).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + if result: + row = result.first() + print( + f"[OK] count={row.get('task_count')} sum={row.get('total_hours')} " + f"avg={row.get('avg_hours')} min={row.get('min_hours')} max={row.get('max_hours')}" + ) + except Exception as e: + print(f"[WARN] aggregate failed: {e}") + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(aggregate group-by project).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] Hours per project ({len(result)} groups):") + for r in result: + print( + f" {r.get('project_code', ''):<10s} " + f"Tasks={r.get('task_count')} " + f"Hours={r.get('total_hours')}" + ) + except Exception as e: + print(f"[WARN] group-by aggregate failed: {e}") + + # =================================================================== + # 10. Built-in system tables + # =================================================================== + heading(10, "Built-In System Tables (account → contact Join)") + xml = """ + + + + + + + + + """ + log_call("await client.query.fetchxml(account → contact inner join).execute()") + try: + result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] {len(result)} account-contact pairs:") + for r in result: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('c.fullname', '')}") + except Exception as e: + print(f"[INFO] No account-contact data in this org: {e}") + + finally: + heading(11, "Cleanup") + for tbl in [task_table, project_table]: + log_call(f"await client.tables.delete('{tbl}')") + try: + await backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + if "404" in str(ex) or (isinstance(ex, MetadataError) and "not found" in str(ex).lower()): + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("Async FetchXML Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/file_upload.py b/examples/aio/advanced/file_upload.py new file mode 100644 index 00000000..e44f6a7c --- /dev/null +++ b/examples/aio/advanced/file_upload.py @@ -0,0 +1,361 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async File Upload Example + +Async equivalent of examples/advanced/file_upload.py. + +This example demonstrates file upload capabilities using the async +PowerPlatform-Dataverse-Client SDK with automatic chunking for large files. + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import hashlib +import sys +import traceback +from pathlib import Path + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +ATTRIBUTE_VISIBILITY_DELAYS = (0, 3, 10, 20, 35, 50, 70, 90, 120) + +# --- Helpers --- + +_FILE_HASH_CACHE: dict = {} + + +def file_sha256(path: Path): + """Return (hex_digest, size_bytes) for the file, with caching.""" + try: + cached = _FILE_HASH_CACHE.get(path) + if cached: + return cached + h = hashlib.sha256() + size = 0 + with path.open("rb") as f: + for chunk in iter(lambda: f.read(1024 * 1024), b""): + size += len(chunk) + h.update(chunk) + result = (h.hexdigest(), size) + _FILE_HASH_CACHE[path] = result + return result + except Exception: # noqa: BLE001 + return None, None + + +def generate_test_file(size_mb: int = 10) -> Path: + """Generate a dummy text file of the specified size for testing.""" + test_file = Path(__file__).resolve().parent / f"test_dummy_{size_mb}mb.txt" + target_size = size_mb * 1024 * 1024 + + line = b"The quick brown fox jumps over the lazy dog. " * 2 + b"\n" + with test_file.open("wb") as f: + written = 0 + while written < target_size: + chunk = line * min(1000, (target_size - written) // len(line) + 1) + chunk = chunk[: target_size - written] + f.write(chunk) + written += len(chunk) + + print({"test_file_generated": str(test_file), "size_mb": test_file.stat().st_size / (1024 * 1024)}) + return test_file + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an async operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" [INFO] Backoff succeeded after {retry_count} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); waited {total_delay}s total.") + raise last + + +# --- Table ensure --- +TABLE_SCHEMA_NAME = "new_FileSample" + + +async def ensure_table(client) -> dict: + """Get or create the demo table.""" + existing = await backoff(lambda: client.tables.get(TABLE_SCHEMA_NAME)) + if existing: + print({"table": TABLE_SCHEMA_NAME, "existed": True}) + return existing + info = await backoff(lambda: client.tables.create(TABLE_SCHEMA_NAME, {"new_Title": "string"})) + print({"table": TABLE_SCHEMA_NAME, "existed": False, "metadata_id": info.get("metadata_id")}) + return info + + +# --- Main --- + + +async def main(): + entered = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not entered: + print("No URL entered; exiting.") + sys.exit(1) + base_url = entered.rstrip("/") + + # Mode selection (numeric): + # 1 = small (single PATCH <128MB) + # 2 = chunk (streaming for any size) + # 3 = all (small + chunk) + mode_raw = input("Choose mode: 1) small 2) chunk 3) all [default 3]: ").strip() + if not mode_raw: + mode_raw = "3" + if mode_raw not in {"1", "2", "3"}: + print({"invalid_mode": mode_raw, "fallback": 3}) + mode_raw = "3" + mode_int = int(mode_raw) + run_small = mode_int in (1, 3) + run_chunk = mode_int in (2, 3) + + delete_record_choice = input("Delete the created record at end? (Y/n): ").strip() or "y" + cleanup_record = delete_record_choice.lower() in ("y", "yes", "true", "1") + + delete_table_choice = input("Delete the table at end? (y/N): ").strip() or "n" + cleanup_table = delete_table_choice.lower() in ("y", "yes", "true", "1") + + credential = AsyncInteractiveBrowserCredential() + + # Generate test files before entering the async context + generated_10mb = generate_test_file(10) + generated_8mb = generate_test_file(8) + + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + + # --------------------------- Table ensure --------------------------- + try: + table_info = await ensure_table(client) + except Exception: # noqa: BLE001 + print("Table ensure failed:") + traceback.print_exc() + sys.exit(1) + + entity_set = table_info.get("entity_set_name") + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + name_attr = f"{attr_prefix}_name" + small_file_attr_schema = f"{attr_prefix}_SmallDocument" + chunk_file_attr_schema = f"{attr_prefix}_ChunkDocument" + + # --------------------------- Record create --------------------------- + record_id = None + try: + payload = {name_attr: "Async File Sample Record"} + print({"call": f"client.records.create('{table_schema_name}', payload)"}) + record_id = await backoff(lambda: client.records.create(table_schema_name, payload)) + print({"record_created": True, "id": record_id, "table schema name": table_schema_name}) + except Exception as e: # noqa: BLE001 + print({"record_created": False, "error": str(e)}) + sys.exit(1) + + if not record_id: + print("No record id; aborting upload.") + sys.exit(1) + + # --------------------------- Small single-request upload --------------------------- + if run_small: + print("Small single-request upload demo:") + try: + src_hash, small_file_size = file_sha256(generated_10mb) + + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=small_file_attr_schema, + path=str(generated_10mb), + mode="small", + ) + ) + print({"small_upload_completed": True, "small_source_size": small_file_size}) + + # Download and verify via internal OData client + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{small_file_attr_schema.lower()}/$value" + resp = await od._request("get", dl_url) + content = await resp.read() if hasattr(resp, "read") else (resp.content or b"") + + downloaded_hash = hashlib.sha256(content).hexdigest() if content else None + hash_match = (downloaded_hash == src_hash) if (downloaded_hash and src_hash) else None + print( + { + "small_file_source_size": small_file_size, + "small_file_download_size": len(content), + "small_file_size_match": len(content) == small_file_size, + "small_file_source_sha256_prefix": src_hash[:16] if src_hash else None, + "small_file_download_sha256_prefix": downloaded_hash[:16] if downloaded_hash else None, + "small_file_hash_match": hash_match, + } + ) + + # Replace with 8MB file + print("Small single-request upload demo - REPLACE with 8MB file:") + replace_hash, replace_size = file_sha256(generated_8mb) + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=small_file_attr_schema, + path=str(generated_8mb), + mode="small", + if_none_match=False, + ) + ) + print({"small_replace_upload_completed": True, "small_replace_source_size": replace_size}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{small_file_attr_schema.lower()}/$value" + resp_r = await od._request("get", dl_url) + content_r = await resp_r.read() if hasattr(resp_r, "read") else (resp_r.content or b"") + + dl_hash_r = hashlib.sha256(content_r).hexdigest() if content_r else None + hash_match_r = (dl_hash_r == replace_hash) if (dl_hash_r and replace_hash) else None + print( + { + "small_replace_source_size": replace_size, + "small_replace_download_size": len(content_r), + "small_replace_size_match": len(content_r) == replace_size, + "small_replace_source_sha256_prefix": replace_hash[:16] if replace_hash else None, + "small_replace_download_sha256_prefix": dl_hash_r[:16] if dl_hash_r else None, + "small_replace_hash_match": hash_match_r, + } + ) + except Exception as ex: # noqa: BLE001 + print({"single_upload_failed": str(ex)}) + + # --------------------------- Chunk (streaming) upload --------------------------- + if run_chunk: + print("Streaming chunk upload demo (mode='chunk'):") + try: + src_hash_chunk, src_size_chunk = file_sha256(generated_10mb) + + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=chunk_file_attr_schema, + path=str(generated_10mb), + mode="chunk", + ) + ) + print({"chunk_upload_completed": True}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{chunk_file_attr_schema.lower()}/$value" + resp = await od._request("get", dl_url) + content_chunk = await resp.read() if hasattr(resp, "read") else (resp.content or b"") + + dst_hash_chunk = hashlib.sha256(content_chunk).hexdigest() if content_chunk else None + hash_match_chunk = ( + (dst_hash_chunk == src_hash_chunk) if (dst_hash_chunk and src_hash_chunk) else None + ) + print( + { + "chunk_source_size": src_size_chunk, + "chunk_download_size": len(content_chunk), + "chunk_size_match": len(content_chunk) == src_size_chunk, + "chunk_source_sha256_prefix": src_hash_chunk[:16] if src_hash_chunk else None, + "chunk_download_sha256_prefix": dst_hash_chunk[:16] if dst_hash_chunk else None, + "chunk_hash_match": hash_match_chunk, + } + ) + + # Replace with 8MB file + print("Streaming chunk upload demo - REPLACE with 8MB file:") + replace_hash_c, replace_size_c = file_sha256(generated_8mb) + await backoff( + lambda: client.files.upload( + table=table_schema_name, + record_id=record_id, + file_column=chunk_file_attr_schema, + path=str(generated_8mb), + mode="chunk", + if_none_match=False, + ) + ) + print({"chunk_replace_upload_completed": True}) + + async with client._scoped_odata() as od: + dl_url = f"{od.api}/{entity_set}({record_id})/{chunk_file_attr_schema.lower()}/$value" + resp_rc = await od._request("get", dl_url) + content_rc = await resp_rc.read() if hasattr(resp_rc, "read") else (resp_rc.content or b"") + + dl_hash_rc = hashlib.sha256(content_rc).hexdigest() if content_rc else None + hash_match_rc = (dl_hash_rc == replace_hash_c) if (dl_hash_rc and replace_hash_c) else None + print( + { + "chunk_replace_source_size": replace_size_c, + "chunk_replace_download_size": len(content_rc), + "chunk_replace_size_match": len(content_rc) == replace_size_c, + "chunk_replace_source_sha256_prefix": replace_hash_c[:16] if replace_hash_c else None, + "chunk_replace_download_sha256_prefix": dl_hash_rc[:16] if dl_hash_rc else None, + "chunk_replace_hash_match": hash_match_rc, + } + ) + except Exception as ex: # noqa: BLE001 + print({"chunk_upload_failed": str(ex)}) + + # --------------------------- Cleanup --------------------------- + if cleanup_record and record_id: + try: + print({"call": f"client.records.delete('{table_schema_name}', '{record_id}')"}) + await backoff(lambda: client.records.delete(table_schema_name, record_id)) + print({"record_deleted": True}) + except Exception as e: # noqa: BLE001 + print({"record_deleted": False, "error": str(e)}) + else: + print({"record_deleted": False, "reason": "user opted to keep"}) + + if cleanup_table: + try: + print({"call": f"client.tables.delete('{TABLE_SCHEMA_NAME}')"}) + await backoff(lambda: client.tables.delete(TABLE_SCHEMA_NAME)) + print({"table_deleted": True}) + except Exception as e: # noqa: BLE001 + print({"table_deleted": False, "error": str(e)}) + else: + print({"table_deleted": False, "reason": "user opted to keep"}) + finally: + await credential.close() + + # Clean up generated test files + for f in [generated_10mb, generated_8mb]: + if f and f.exists(): + try: + f.unlink() + print({"test_file_deleted": True, "path": str(f)}) + except Exception as e: # noqa: BLE001 + print({"test_file_deleted": False, "error": str(e)}) + + print("Done.") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/prodev_quick_start.py b/examples/aio/advanced/prodev_quick_start.py new file mode 100644 index 00000000..4cc953a8 --- /dev/null +++ b/examples/aio/advanced/prodev_quick_start.py @@ -0,0 +1,475 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Pro-Dev Quick Start + +Async equivalent of examples/advanced/prodev_quick_start.py. + +A developer-focused example that demonstrates the full async SDK lifecycle: +install, authenticate, create a system with 4 related tables, populate +data, query it, and clean up -- all in a single script. + +What this example covers: + 1) SDK installation and authentication + 2) Create 4 custom tables concurrently with asyncio.gather() + 3) Create columns and relationships between tables + 4) Populate with sample data using async DataFrame CRUD + 5) Query and join data across tables + 6) Clean up (delete tables) + +Prerequisites: + pip install PowerPlatform-Dataverse-Client + pip install azure-identity +""" + +import asyncio +import sys +import uuid +import warnings +from pathlib import Path + +# Suppress MSAL advisory about response_mode (third-party library, not actionable here) +warnings.filterwarnings("ignore", message="response_mode=.*form_post", category=UserWarning) + +import pandas as pd + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.filters import col + +# -- Table schema names -- +SUFFIX = uuid.uuid4().hex[:6] +TABLE_CUSTOMER = f"new_DemoCustomer{SUFFIX}" +TABLE_PROJECT = f"new_DemoProject{SUFFIX}" +TABLE_TASK = f"new_DemoTask{SUFFIX}" +TABLE_TIMEENTRY = f"new_DemoTimeEntry{SUFFIX}" + +# -- Output folder for exported data -- +_SCRIPT_DIR = Path(__file__).resolve().parent +OUTPUT_DIR = _SCRIPT_DIR / "prodev_output" + + +async def main(): + """Entry point.""" + print("=" * 60) + print(" DATAVERSE PYTHON SDK -- ASYNC PRO-DEV QUICK START") + print("=" * 60) + print() + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("[ERR] No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + print("[INFO] Authenticating via browser (Azure Identity)...") + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient(base_url, credential) as client: + try: + await run_demo(client) + except Exception as e: + print(f"\n[ERR] {e}") + print("[INFO] Attempting cleanup...") + await cleanup(client) + raise + finally: + await credential.close() + + +async def run_demo(client): + """Run the full async pro-dev demo pipeline.""" + OUTPUT_DIR.mkdir(exist_ok=True) + print(f"[INFO] Output folder: {OUTPUT_DIR.resolve()}") + + # -- Step 1: Create 4 tables concurrently -- + primary_name_col, primary_id_col = await step1_create_tables(client) + + # -- Step 2: Create relationships -- + await step2_create_relationships(client) + + # -- Step 3: Populate with sample data -- + customer_ids, project_ids, task_ids = await step3_populate_data(client, primary_name_col) + + # -- Step 4: Query and analyze -- + await step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col) + + # -- Step 5: Update and delete -- + await step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col) + + # -- Step 6: Cleanup -- + do_cleanup = input("\n6. Delete demo tables and cleanup? (Y/n): ").strip() or "y" + if do_cleanup.lower() in ("y", "yes"): + await cleanup(client) + else: + print("[INFO] Tables kept for inspection.") + + print("\n" + "=" * 60) + print("[OK] Async pro-dev quick start demo complete!") + print("=" * 60) + + +# ================================================================ +# Step 1: Create tables (concurrently with asyncio.gather) +# ================================================================ + + +async def step1_create_tables(client): + """Create 4 custom tables sequentially. + + Note: Dataverse holds a metadata customization lock for the duration of + each table-creation request. Concurrent creates (asyncio.gather) trigger + a CustomizationLockException on the server, so tables must be created one + at a time. + """ + print("\n" + "-" * 60) + print("STEP 1: Create 4 custom tables (sequentially)") + print("-" * 60) + + customer_result = await client.tables.create( + TABLE_CUSTOMER, + { + f"{TABLE_CUSTOMER}_Email": "string", + f"{TABLE_CUSTOMER}_Industry": "string", + f"{TABLE_CUSTOMER}_Revenue": "money", + }, + ) + await client.tables.create( + TABLE_PROJECT, + { + f"{TABLE_PROJECT}_Budget": "money", + f"{TABLE_PROJECT}_Status": "string", + f"{TABLE_PROJECT}_StartDate": "datetime", + }, + ) + await client.tables.create( + TABLE_TASK, + { + f"{TABLE_TASK}_Priority": "integer", + f"{TABLE_TASK}_Status": "string", + f"{TABLE_TASK}_EstimatedHours": "decimal", + }, + ) + await client.tables.create( + TABLE_TIMEENTRY, + { + f"{TABLE_TIMEENTRY}_Hours": "decimal", + f"{TABLE_TIMEENTRY}_Date": "datetime", + f"{TABLE_TIMEENTRY}_Description": "string", + }, + ) + + primary_name_col = customer_result.primary_name_attribute + primary_id_col = customer_result.primary_id_attribute + print(f"[OK] Created table: {TABLE_CUSTOMER} (name: {primary_name_col}, id: {primary_id_col})") + print(f"[OK] Created table: {TABLE_PROJECT}") + print(f"[OK] Created table: {TABLE_TASK}") + print(f"[OK] Created table: {TABLE_TIMEENTRY}") + print(f"[OK] All 4 tables created (suffix: {SUFFIX})") + + return primary_name_col, primary_id_col + + +# ================================================================ +# Step 2: Create relationships +# ================================================================ + + +async def step2_create_relationships(client): + """Create relationships between the 4 tables using lookup fields.""" + print("\n" + "-" * 60) + print("STEP 2: Create relationships (lookup fields)") + print("-" * 60) + + # Relationships must be created sequentially -- Dataverse rejects + # concurrent metadata writes to related tables. + await client.tables.create_lookup_field( + referencing_table=TABLE_PROJECT.lower(), + lookup_field_name=f"{TABLE_PROJECT}_CustomerId", + referenced_table=TABLE_CUSTOMER.lower(), + display_name="Customer", + ) + print(f"[OK] {TABLE_CUSTOMER} 1:N {TABLE_PROJECT}") + + await client.tables.create_lookup_field( + referencing_table=TABLE_TASK.lower(), + lookup_field_name=f"{TABLE_TASK}_ProjectId", + referenced_table=TABLE_PROJECT.lower(), + display_name="Project", + ) + print(f"[OK] {TABLE_PROJECT} 1:N {TABLE_TASK}") + + await client.tables.create_lookup_field( + referencing_table=TABLE_TIMEENTRY.lower(), + lookup_field_name=f"{TABLE_TIMEENTRY}_TaskId", + referenced_table=TABLE_TASK.lower(), + display_name="Task", + ) + print(f"[OK] {TABLE_TASK} 1:N {TABLE_TIMEENTRY}") + print("[OK] 3 lookup relationships created (Customer -> Project -> Task -> TimeEntry)") + + +# ================================================================ +# Step 3: Populate with sample data +# ================================================================ + + +async def step3_populate_data(client, primary_name_col): + """Create sample records using client.dataframe.create().""" + print("\n" + "-" * 60) + print("STEP 3: Populate with sample data (async DataFrame CRUD)") + print("-" * 60) + + # -- Customers -- + name_col = primary_name_col + customers_df = pd.DataFrame( + [ + { + name_col: "Contoso Ltd", + f"{TABLE_CUSTOMER}_Email": "info@contoso.com", + f"{TABLE_CUSTOMER}_Industry": "Technology", + f"{TABLE_CUSTOMER}_Revenue": 5000000, + }, + { + name_col: "Fabrikam Inc", + f"{TABLE_CUSTOMER}_Email": "contact@fabrikam.com", + f"{TABLE_CUSTOMER}_Industry": "Manufacturing", + f"{TABLE_CUSTOMER}_Revenue": 12000000, + }, + { + name_col: "Northwind Traders", + f"{TABLE_CUSTOMER}_Email": "sales@northwind.com", + f"{TABLE_CUSTOMER}_Industry": "Retail", + f"{TABLE_CUSTOMER}_Revenue": 3000000, + }, + ] + ) + customer_ids = await client.dataframe.create(TABLE_CUSTOMER, customers_df) + customers_df["id"] = customer_ids + print(f"[OK] Created {len(customers_df)} customers") + + # -- Projects (linked to customers via lookup) -- + customer_lookup = f"{TABLE_PROJECT}_CustomerId@odata.bind" + customer_info = await client.tables.get(TABLE_CUSTOMER) + customer_set = customer_info.get("entity_set_name") if customer_info else TABLE_CUSTOMER.lower() + "s" + projects_df = pd.DataFrame( + [ + { + name_col: "Cloud Migration", + f"{TABLE_PROJECT}_Budget": 250000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-15"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + { + name_col: "ERP Upgrade", + f"{TABLE_PROJECT}_Budget": 500000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-02-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[1]})", + }, + { + name_col: "POS Modernization", + f"{TABLE_PROJECT}_Budget": 150000, + f"{TABLE_PROJECT}_Status": "Planning", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-03-01"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[2]})", + }, + { + name_col: "Data Analytics Platform", + f"{TABLE_PROJECT}_Budget": 180000, + f"{TABLE_PROJECT}_Status": "Active", + f"{TABLE_PROJECT}_StartDate": pd.Timestamp("2026-01-20"), + customer_lookup: f"/{customer_set}({customer_ids.iloc[0]})", + }, + ] + ) + project_ids = await client.dataframe.create(TABLE_PROJECT, projects_df) + projects_df["id"] = project_ids + print(f"[OK] Created {len(projects_df)} projects across 3 customers") + + # -- Tasks (linked to projects) -- + task_names = [ + ("Infrastructure Setup", 1, "In Progress", 40), + ("Data Assessment", 2, "Not Started", 20), + ("Testing & QA", 1, "Not Started", 60), + ("Requirements Gathering", 1, "Complete", 30), + ("Development Sprint 1", 1, "In Progress", 80), + ("User Training", 3, "Not Started", 16), + ] + project_assignment = [0, 0, 0, 1, 1, 2] + + project_info = await client.tables.get(TABLE_PROJECT) + project_set = project_info.get("entity_set_name") if project_info else TABLE_PROJECT.lower() + "s" + project_lookup = f"{TABLE_TASK}_ProjectId@odata.bind" + + tasks_data = [ + { + name_col: task_name, + f"{TABLE_TASK}_Priority": priority, + f"{TABLE_TASK}_Status": status, + f"{TABLE_TASK}_EstimatedHours": hours, + project_lookup: f"/{project_set}({project_ids.iloc[project_assignment[i]]})", + } + for i, (task_name, priority, status, hours) in enumerate(task_names) + ] + + tasks_df = pd.DataFrame(tasks_data) + task_ids = await client.dataframe.create(TABLE_TASK, tasks_df) + tasks_df["id"] = task_ids + print(f"[OK] Created {len(tasks_df)} tasks across 4 projects") + + print( + f"\n Total records: {len(customers_df) + len(projects_df) + len(tasks_df)} " + f"({len(customers_df)} customers, {len(projects_df)} projects, {len(tasks_df)} tasks)" + ) + + return customer_ids, project_ids, task_ids + + +# ================================================================ +# Step 4: Query and analyze data +# ================================================================ + + +async def step4_query_and_analyze(client, customer_ids, primary_name_col, primary_id_col): + """Query data and demonstrate DataFrame analysis.""" + print("\n" + "-" * 60) + print("STEP 4: Query and analyze data") + print("-" * 60) + + name_attr = primary_name_col + + # Query projects and tasks concurrently + project_result, task_result = await asyncio.gather( + client.query.builder(TABLE_PROJECT) + .select(name_attr, f"{TABLE_PROJECT}_Budget", f"{TABLE_PROJECT}_Status") + .execute(), + client.query.builder(TABLE_TASK) + .select(name_attr, f"{TABLE_TASK}_Priority", f"{TABLE_TASK}_Status", f"{TABLE_TASK}_EstimatedHours") + .execute(), + ) + + projects = project_result.to_dataframe() + tasks = task_result.to_dataframe() + + print(f"\n All projects ({len(projects)} rows):") + print(f"{projects.to_string(index=False)}") + + print(f"\n All tasks ({len(tasks)} rows):") + print(f"{tasks.to_string(index=False)}") + + # -- DataFrame analysis -- + hours_col = f"{TABLE_TASK}_EstimatedHours" + status_col = f"{TABLE_TASK}_Status" + budget_col = f"{TABLE_PROJECT}_Budget" + + if hours_col in tasks.columns: + print(f"\n Task hours summary:") + print(f" Total estimated hours: {tasks[hours_col].sum():.0f}") + print(f" Average per task: {tasks[hours_col].mean():.1f}") + print(f" Max single task: {tasks[hours_col].max():.0f}") + + if status_col in tasks.columns: + print(f"\n Tasks by status:") + for status, count in tasks[status_col].value_counts().items(): + print(f" {status}: {count}") + + if budget_col in projects.columns: + print(f"\n Project budget summary:") + print(f" Total budget: ${projects[budget_col].sum():,.0f}") + print(f" Average budget: ${projects[budget_col].mean():,.0f}") + + # Fetch single customer record by ID + first_id = customer_ids.iloc[0] + single_result = await client.query.builder(TABLE_CUSTOMER).where(col(primary_id_col) == first_id).execute() + single = single_result.to_dataframe() + print(f"\n Single customer record (by ID):") + print(f"{single.to_string(index=False)}") + + # -- Export query results to CSV -- + projects.to_csv(OUTPUT_DIR / "projects.csv", index=False) + tasks.to_csv(OUTPUT_DIR / "tasks.csv", index=False) + single.to_csv(OUTPUT_DIR / "single_customer.csv", index=False) + print(f"\n[OK] Exported query results to {OUTPUT_DIR}/") + + +# ================================================================ +# Step 5: Update and delete records +# ================================================================ + + +async def step5_update_and_delete(client, task_ids, primary_name_col, primary_id_col): + """Demonstrate update and delete with DataFrames.""" + print("\n" + "-" * 60) + print("STEP 5: Update and delete records") + print("-" * 60) + + status_col = f"{TABLE_TASK}_Status" + + # Update: mark first two tasks as "Complete" + update_df = pd.DataFrame( + { + primary_id_col: [task_ids.iloc[0], task_ids.iloc[1]], + status_col: ["Complete", "Complete"], + } + ) + await client.dataframe.update(TABLE_TASK, update_df, id_column=primary_id_col) + print(f"[OK] Updated 2 tasks to 'Complete'") + + # Delete: remove the last task + delete_ids = pd.Series([task_ids.iloc[-1]]) + await client.dataframe.delete(TABLE_TASK, delete_ids) + print(f"[OK] Deleted 1 task") + + # Verify + result = await client.query.builder(TABLE_TASK).select(primary_name_col, status_col).execute() + remaining = result.to_dataframe() + print(f"\n Remaining tasks ({len(remaining)}):") + print(f"{remaining.to_string(index=False)}") + + +# ================================================================ +# Cleanup +# ================================================================ + + +async def cleanup(client): + """Delete all demo tables. + + Tables must be deleted leaf-to-root (TimeEntry → Task → Project → Customer) + because each table holds a lookup field referencing the next. Dataverse may + also return transient SQL-deadlock errors during metadata operations, so we + retry failed deletions until all tables are gone or no further progress is + made. + """ + print("\n" + "-" * 60) + print("CLEANUP: Removing demo tables") + print("-" * 60) + + remaining = [TABLE_TIMEENTRY, TABLE_TASK, TABLE_PROJECT, TABLE_CUSTOMER] + while remaining: + failed = [] + for table in remaining: + try: + await client.tables.delete(table) + print(f"[OK] Deleted table: {table}") + except Exception as e: + failed.append((table, e)) + + if len(failed) == len(remaining): + # No progress — report and stop to avoid an infinite loop. + for table, e in failed: + print(f"[WARN] Could not delete {table}: {e}") + break + + remaining = [t for t, _ in failed] + + print("[OK] Cleanup complete") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/relationships.py b/examples/aio/advanced/relationships.py new file mode 100644 index 00000000..a4f9ecff --- /dev/null +++ b/examples/aio/advanced/relationships.py @@ -0,0 +1,401 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async Relationship Management Example for Dataverse SDK. + +Async equivalent of examples/advanced/relationships.py. + +This example demonstrates: +- Creating one-to-many relationships using the core SDK API +- Creating lookup fields using the convenience method +- Creating many-to-many relationships +- Querying and deleting relationships +- Working with relationship metadata types + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client +- pip install azure-identity +""" + +import asyncio +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, +) + + +# Simple logging helper +def log_call(description): + print(f"\n-> {description}") + + +async def delete_relationship_if_exists(client, schema_name): + """Delete a relationship by schema name if it exists.""" + rel = await client.tables.get_relationship(schema_name) + if rel: + rel_id = rel.relationship_id + if rel_id: + await client.tables.delete_relationship(rel_id) + print(f" (Cleaned up existing relationship: {schema_name})") + return True + return False + + +async def cleanup_previous_run(client): + """Clean up any resources from a previous run to make the example idempotent.""" + print("\n-> Checking for resources from previous runs...") + + # Known relationship names created by this example + relationships = [ + "new_Department_Employee", + "contact_new_employee_new_ManagerId", + "new_employee_project", + ] + + # Known table names created by this example + tables = ["new_Employee", "new_Department", "new_Project"] + + # Delete relationships first (required before tables can be deleted) + for rel_name in relationships: + try: + await delete_relationship_if_exists(client, rel_name) + except Exception as e: + print(f" [WARN] Could not delete relationship {rel_name}: {e}") + + # Delete tables + for table_name in tables: + try: + if await client.tables.get(table_name): + await client.tables.delete(table_name) + print(f" (Cleaned up existing table: {table_name})") + except Exception as e: + print(f" [WARN] Could not delete table {table_name}: {e}") + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry helper with exponential backoff.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + retry_count = attempts - 1 + print(f" * Backoff succeeded after {retry_count} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: # noqa: BLE001 + last = ex + continue + if last: + if attempts: + retry_count = max(attempts - 1, 0) + print(f" [WARN] Backoff exhausted after {retry_count} retry(s); waited {total_delay}s total.") + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK - Async Relationship Management Example") + print("=" * 80) + + # ============================================================================ + # 1. SETUP & AUTHENTICATION + # ============================================================================ + print("\n" + "=" * 80) + print("1. Setup & Authentication") + print("=" * 80) + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_example(client) + finally: + await credential.close() + + +async def _run_example(client): + # Initialize relationship IDs to None for cleanup safety + rel_id_1 = None + rel_id_2 = None + rel_id_3 = None + + # ============================================================================ + # 2. CLEANUP PREVIOUS RUN (Idempotency) + # ============================================================================ + print("\n" + "=" * 80) + print("2. Cleanup Previous Run (Idempotency)") + print("=" * 80) + + await cleanup_previous_run(client) + + # ============================================================================ + # 3. CREATE SAMPLE TABLES + # ============================================================================ + print("\n" + "=" * 80) + print("3. Create Sample Tables") + print("=" * 80) + + # Create a parent table (Department) + log_call("Creating 'new_Department' table") + + dept_table = await backoff( + lambda: client.tables.create( + "new_Department", + { + "new_DepartmentCode": "string", + "new_Budget": "decimal", + }, + ) + ) + print(f"[OK] Created table: {dept_table['table_schema_name']}") + + # Create a child table (Employee) + log_call("Creating 'new_Employee' table") + + emp_table = await backoff( + lambda: client.tables.create( + "new_Employee", + { + "new_EmployeeNumber": "string", + "new_Salary": "decimal", + }, + ) + ) + print(f"[OK] Created table: {emp_table['table_schema_name']}") + + # Create a project table for many-to-many example + log_call("Creating 'new_Project' table") + + proj_table = await backoff( + lambda: client.tables.create( + "new_Project", + { + "new_ProjectCode": "string", + "new_StartDate": "datetime", + }, + ) + ) + print(f"[OK] Created table: {proj_table['table_schema_name']}") + + # ============================================================================ + # 4. CREATE ONE-TO-MANY RELATIONSHIP (Core SDK API) + # ============================================================================ + print("\n" + "=" * 80) + print("4. Create One-to-Many Relationship (Core API)") + print("=" * 80) + + log_call("Creating lookup field on Employee referencing Department") + + # Define the lookup attribute metadata + lookup = LookupAttributeMetadata( + schema_name="new_DepartmentId", + display_name=Label(localized_labels=[LocalizedLabel(label="Department", language_code=1033)]), + required_level="None", + ) + + # Define the relationship metadata + relationship = OneToManyRelationshipMetadata( + schema_name="new_Department_Employee", + referenced_entity=dept_table["table_logical_name"], + referencing_entity=emp_table["table_logical_name"], + referenced_attribute=f"{dept_table['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + merge=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + + # Create the relationship + result = await backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) + ) + + print(f"[OK] Created relationship: {result.relationship_schema_name}") + print(f" Lookup field: {result.lookup_schema_name}") + print(f" Relationship ID: {result.relationship_id}") + + rel_id_1 = result.relationship_id + + # ============================================================================ + # 5. CREATE LOOKUP FIELD (Convenience Method) + # ============================================================================ + print("\n" + "=" * 80) + print("5. Create Lookup Field (Convenience Method)") + print("=" * 80) + + log_call("Creating lookup field on Employee referencing Contact as Manager") + + # Use the convenience method for simpler scenarios + # An Employee has a Manager (who is a Contact in the system) + result2 = await backoff( + lambda: client.tables.create_lookup_field( + referencing_table=emp_table["table_logical_name"], + lookup_field_name="new_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The employee's direct manager", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + + print(f"[OK] Created lookup using convenience method: {result2.lookup_schema_name}") + print(f" Relationship: {result2.relationship_schema_name}") + + rel_id_2 = result2.relationship_id + + # ============================================================================ + # 6. CREATE MANY-TO-MANY RELATIONSHIP + # ============================================================================ + print("\n" + "=" * 80) + print("6. Create Many-to-Many Relationship") + print("=" * 80) + + log_call("Creating M:N relationship between Employee and Project") + + # Define many-to-many relationship + m2m_relationship = ManyToManyRelationshipMetadata( + schema_name="new_employee_project", + entity1_logical_name=emp_table["table_logical_name"], + entity2_logical_name=proj_table["table_logical_name"], + ) + + result3 = await backoff( + lambda: client.tables.create_many_to_many_relationship( + relationship=m2m_relationship, + ) + ) + + print(f"[OK] Created M:N relationship: {result3.relationship_schema_name}") + print(f" Relationship ID: {result3.relationship_id}") + + rel_id_3 = result3.relationship_id + + # ============================================================================ + # 7. QUERY RELATIONSHIP METADATA + # ============================================================================ + print("\n" + "=" * 80) + print("7. Query Relationship Metadata") + print("=" * 80) + + log_call("Retrieving 1:N relationship by schema name") + + rel_metadata = await client.tables.get_relationship("new_Department_Employee") + if rel_metadata: + print(f"[OK] Found relationship: {rel_metadata.relationship_schema_name}") + print(f" Type: {rel_metadata.relationship_type}") + print(f" Referenced Entity: {rel_metadata.referenced_entity}") + print(f" Referencing Entity: {rel_metadata.referencing_entity}") + else: + print(" Relationship not found") + + log_call("Retrieving M:N relationship by schema name") + + m2m_metadata = await client.tables.get_relationship("new_employee_project") + if m2m_metadata: + print(f"[OK] Found relationship: {m2m_metadata.relationship_schema_name}") + print(f" Type: {m2m_metadata.relationship_type}") + print(f" Entity 1: {m2m_metadata.entity1_logical_name}") + print(f" Entity 2: {m2m_metadata.entity2_logical_name}") + else: + print(" Relationship not found") + + # ============================================================================ + # 8. CLEANUP + # ============================================================================ + print("\n" + "=" * 80) + print("8. Cleanup") + print("=" * 80) + + cleanup = input("\nDelete created relationships and tables? (y/n): ").strip().lower() + + if cleanup == "y": + # Delete relationships first (required before deleting tables) + log_call("Deleting relationships") + try: + if rel_id_1: + await backoff(lambda: client.tables.delete_relationship(rel_id_1)) + print(f" [OK] Deleted relationship: new_Department_Employee") + except Exception as e: + print(f" [WARN] Error deleting relationship 1: {e}") + + try: + if rel_id_2: + await backoff(lambda: client.tables.delete_relationship(rel_id_2)) + print(f" [OK] Deleted relationship: contact->employee (Manager)") + except Exception as e: + print(f" [WARN] Error deleting relationship 2: {e}") + + try: + if rel_id_3: + await backoff(lambda: client.tables.delete_relationship(rel_id_3)) + print(f" [OK] Deleted relationship: new_employee_project") + except Exception as e: + print(f" [WARN] Error deleting relationship 3: {e}") + + # Delete tables + log_call("Deleting tables") + for table_name in ["new_Employee", "new_Department", "new_Project"]: + try: + await backoff(lambda name=table_name: client.tables.delete(name)) + print(f" [OK] Deleted table: {table_name}") + except Exception as e: + print(f" [WARN] Error deleting {table_name}: {e}") + + print("\n[OK] Cleanup complete") + else: + print("\nSkipping cleanup. Remember to manually delete:") + print(" - Relationships: new_Department_Employee, contact->employee (Manager), new_employee_project") + print(" - Tables: new_Employee, new_Department, new_Project") + + print("\n" + "=" * 80) + print("Example Complete!") + print("=" * 80) + + +if __name__ == "__main__": + try: + asyncio.run(main()) + except KeyboardInterrupt: + print("\n\nExample interrupted by user.") + sys.exit(1) + except Exception as e: + print(f"\n\nError: {e}") + import traceback + + traceback.print_exc() + sys.exit(1) diff --git a/examples/aio/advanced/sql_examples.py b/examples/aio/advanced/sql_examples.py new file mode 100644 index 00000000..4edd635c --- /dev/null +++ b/examples/aio/advanced/sql_examples.py @@ -0,0 +1,925 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async end-to-end SQL query examples -- pure SQL workflows in Dataverse. + +Async equivalent of examples/advanced/sql_examples.py. + +This example demonstrates everything a SQL developer can do through the +Python SDK's ``client.query.sql()`` and ``client.dataframe.sql()`` methods. + +See examples/advanced/sql_examples.py for the complete capability reference. + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client azure-identity +""" + +import asyncio +import sys +import json +from collections import defaultdict +from enum import IntEnum + +import pandas as pd +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def log_call(description): + print(f"\n-> {description}") + + +def heading(section_num, title): + print(f"\n{'=' * 80}") + print(f"{section_num}. {title}") + print("=" * 80) + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry an async operation with exponential back-off.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} " f"retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print( + f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total." + f"\n [ERROR] {last}" + ) + raise last + + +class Region(IntEnum): + NORTH = 1 + SOUTH = 2 + EAST = 3 + WEST = 4 + + +async def main(): + print("=" * 80) + print("Dataverse SDK -- Async SQL End-to-End (Pure SQL Workflows)") + print("=" * 80) + + heading(1, "Setup & Authentication") + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_examples(client) + finally: + await credential.close() + + +async def _run_examples(client): + parent_table = "new_SQLDemoTeam" + child_table = "new_SQLDemoTask" + + # ================================================================== + # 2. Seed demo data (SDK writes -- SQL is read-only) + # ================================================================== + heading(2, "Seed Demo Data (SDK Writes -- SQL Is Read-Only)") + print( + "[INFO] SQL is read-only (no INSERT/UPDATE/DELETE). We use the SDK's\n" + "records namespace to seed data, then query it all via SQL." + ) + + log_call(f"client.tables.get('{parent_table}')") + if await client.tables.get(parent_table): + print(f"[OK] Table already exists: {parent_table}") + else: + log_call(f"client.tables.create('{parent_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + parent_table, + { + "new_Code": "string", + "new_Region": Region, + "new_Budget": "decimal", + "new_Active": "bool", + }, + ) + ) + print(f"[OK] Created table: {parent_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {parent_table} (skipped)") + else: + raise + + log_call(f"client.tables.get('{child_table}')") + if await client.tables.get(child_table): + print(f"[OK] Table already exists: {child_table}") + else: + log_call(f"client.tables.create('{child_table}', ...)") + try: + await backoff( + lambda: client.tables.create( + child_table, + { + "new_Title": "string", + "new_Hours": "int", + "new_Done": "bool", + "new_Priority": "int", + }, + ) + ) + print(f"[OK] Created table: {child_table}") + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {child_table} (skipped)") + else: + raise + + # Create lookup so tasks reference teams via JOIN + print("\n[INFO] Creating lookup field so tasks reference teams via JOIN...") + try: + await client.tables.create_lookup_field( + referencing_table=child_table, + lookup_field_name="new_TeamId", + referenced_table=parent_table, + display_name="Team", + ) + print("[OK] Created lookup: new_TeamId on tasks -> teams") + except Exception as e: + msg = str(e).lower() + if "already exists" in msg or "duplicate" in msg or "not unique" in msg: + print("[OK] Lookup already exists (skipped)") + else: + raise + + log_call(f"client.records.create('{parent_table}', [...])") + teams = [ + {"new_Code": "ALPHA", "new_Region": Region.NORTH, "new_Budget": 50000, "new_Active": True}, + {"new_Code": "BRAVO", "new_Region": Region.SOUTH, "new_Budget": 75000, "new_Active": True}, + {"new_Code": "CHARLIE", "new_Region": Region.EAST, "new_Budget": 30000, "new_Active": False}, + {"new_Code": "DELTA", "new_Region": Region.WEST, "new_Budget": 90000, "new_Active": True}, + {"new_Code": "ECHO", "new_Region": Region.NORTH, "new_Budget": 42000, "new_Active": True}, + ] + team_ids = await backoff(lambda: client.records.create(parent_table, teams)) + print(f"[OK] Seeded {len(team_ids)} teams") + + parent_logical = parent_table.lower() + parent_set = f"{parent_logical}s" + try: + tinfo = await client.tables.get(parent_table) + if tinfo: + parent_set = tinfo.get("entity_set_name", parent_set) + except Exception: + pass + + log_call(f"client.records.create('{child_table}', [...])") + tasks = [ + { + "new_Title": "Design mockups", + "new_Hours": 8, + "new_Done": True, + "new_Priority": 2, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[0]})", + }, + { + "new_Title": "Write unit tests", + "new_Hours": 12, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[0]})", + }, + { + "new_Title": "Code review", + "new_Hours": 3, + "new_Done": True, + "new_Priority": 1, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[1]})", + }, + { + "new_Title": "Deploy to staging", + "new_Hours": 5, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[1]})", + }, + { + "new_Title": "Update docs", + "new_Hours": 4, + "new_Done": True, + "new_Priority": 1, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[2]})", + }, + { + "new_Title": "Performance tuning", + "new_Hours": 10, + "new_Done": False, + "new_Priority": 2, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[3]})", + }, + { + "new_Title": "Security audit", + "new_Hours": 6, + "new_Done": False, + "new_Priority": 3, + f"new_TeamId@odata.bind": f"/{parent_set}({team_ids[4]})", + }, + ] + task_ids = await backoff(lambda: client.records.create(child_table, tasks)) + print(f"[OK] Seeded {len(task_ids)} tasks (with team lookups)") + + parent_id_col = f"{parent_logical}id" + + try: + # ============================================================== + # 3. Schema discovery + # ============================================================== + heading(3, "Schema Discovery Before Writing SQL") + log_call(f"client.tables.list_columns('{parent_table}', select=[...])") + columns = await backoff( + lambda: client.tables.list_columns( + parent_table, + select=["LogicalName", "SchemaName", "AttributeType"], + ) + ) + custom_cols = [c for c in columns if c.get("LogicalName", "").startswith("new_")] + print(f"[OK] Custom columns on {parent_table}:") + for col in custom_cols: + print(f" {col['LogicalName']:30s} Type: {col.get('AttributeType', 'N/A')}") + + log_call(f"client.tables.list_table_relationships('{child_table}', ...)") + rels = await backoff( + lambda: client.tables.list_table_relationships( + child_table, + select=["SchemaName"], + ) + ) + print(f"[OK] Relationships on {child_table}: {len(rels)}") + + # ============================================================== + # 4. Basic SELECT + # ============================================================== + heading(4, "Basic SQL -- SELECT Specific Columns") + sql = f"SELECT new_code, new_budget, new_active FROM {parent_table}" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} rows:") + for r in results: + print(f" {r.get('new_code', ''):<12s} Budget={r.get('new_budget')} Active={r.get('new_active')}") + + # ============================================================== + # 5. SELECT * -- Rejected by Design + # ============================================================== + heading(5, "SELECT * -- Rejected by Design") + print( + "SELECT * is deliberately rejected -- not a server workaround,\n" + "but an intentional design decision. Wide entities (e.g. account\n" + "has 307 columns) make SELECT * extremely expensive on shared\n" + "infrastructure. Specify columns explicitly instead.\n" + "Use client.query.sql_columns('account') to discover column names." + ) + from PowerPlatform.Dataverse.core.errors import ValidationError as _VE + + try: + await client.query.sql(f"SELECT * FROM {parent_table}") + print("[UNEXPECTED] SELECT * did not raise -- check SDK version") + except _VE as exc: + print(f"[OK] ValidationError raised as expected: {exc}") + + # ============================================================== + # 6. WHERE clause + # ============================================================== + heading(6, "SQL -- WHERE (=, >, <, IN, IS NULL, BETWEEN)") + sql = f"SELECT new_code, new_budget FROM {parent_table} WHERE new_budget > 40000" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] budget > 40000: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} WHERE new_code IN ('ALPHA', 'DELTA')" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] IN clause: {[r.get('new_code') for r in results]}") + + sql = f"SELECT new_title FROM {child_table} WHERE new_priority IS NOT NULL" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] IS NOT NULL: {len(results)} tasks") + + # ============================================================== + # 7. LIKE + # ============================================================== + heading(7, "SQL -- LIKE Pattern Matching") + sql = f"SELECT new_title FROM {child_table} WHERE new_title LIKE '%test%'" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] LIKE '%test%': {len(results)} matches") + + # ============================================================== + # 8. TOP + ORDER BY + # ============================================================== + heading(8, "SQL -- TOP N + ORDER BY") + sql = f"SELECT TOP 3 new_code, new_budget FROM {parent_table} ORDER BY new_budget DESC" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Top 3 by budget:") + for r in results: + print(f" {r.get('new_code', ''):<12s} Budget={r.get('new_budget')}") + + # ============================================================== + # 9. Aliases + # ============================================================== + heading(9, "SQL -- Table and Column Aliases") + sql = ( + f"SELECT t.new_code AS team_code, t.new_budget AS budget " + f"FROM {parent_table} AS t WHERE t.new_active = 1" + ) + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Aliased results: {len(results)} rows") + + # ============================================================== + # 10. DISTINCT + # ============================================================== + heading(10, "SQL -- DISTINCT") + sql = f"SELECT DISTINCT new_region FROM {parent_table}" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] Distinct regions: {[r.get('new_region') for r in results]}") + + sql = f"SELECT DISTINCT TOP 2 new_region FROM {parent_table}" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] DISTINCT TOP 2: {[r.get('new_region') for r in results]}") + + # ============================================================== + # 11. Aggregates: COUNT, SUM, AVG, MIN, MAX + # ============================================================== + heading(11, "SQL -- Aggregates (All Run on Server)") + sql = ( + f"SELECT COUNT(*) as cnt, SUM(new_budget) as total, " + f"AVG(new_budget) as avg_b, MIN(new_budget) as min_b, " + f"MAX(new_budget) as max_b FROM {parent_table}" + ) + log_call('client.query.sql("SELECT COUNT, SUM, AVG, MIN, MAX...")') + results = await backoff(lambda: client.query.sql(sql)) + if results: + print(f"[OK] {json.dumps(dict(results[0]), indent=2)}") + + # ============================================================== + # 12. GROUP BY + # ============================================================== + heading(12, "SQL -- GROUP BY (Server-Side)") + sql = ( + f"SELECT new_region, COUNT(*) as team_count, " + f"SUM(new_budget) as total_budget " + f"FROM {parent_table} GROUP BY new_region" + ) + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} groups:") + for r in results: + print(f" Region={r.get('new_region')} Count={r.get('team_count')} Total={r.get('total_budget')}") + + # ============================================================== + # 13. INNER JOIN + # ============================================================== + heading(13, "SQL -- INNER JOIN") + print("Use the lookup attribute's logical name (e.g. new_teamid) for JOINs.") + + lookup_col = "new_teamid" # Lookup logical name, NOT _..._value + join_clause = f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_logical}id" + print(f"[INFO] Lookup column: {lookup_col}") + print(f"[INFO] JOIN clause: {join_clause}") + + sql = f"SELECT t.new_code, tk.new_title, tk.new_hours FROM {child_table} tk {join_clause}" + log_call('client.query.sql("...INNER JOIN...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] JOIN: {len(results)} rows") + for r in results[:5]: + print( + f" Team={r.get('new_code', ''):<10s} Task={r.get('new_title', ''):<25s} Hours={r.get('new_hours')}" + ) + except Exception as e: + print(f"[WARN] JOIN failed: {e}") + + # ============================================================== + # 14. LEFT JOIN + # ============================================================== + heading(14, "SQL -- LEFT JOIN") + sql = ( + f"SELECT t.new_code, tk.new_title " + f"FROM {parent_table} t " + f"LEFT JOIN {child_table} tk ON t.{parent_id_col} = tk.{lookup_col}" + ) + log_call('client.query.sql("...LEFT JOIN...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] LEFT JOIN: {len(results)} rows") + except Exception as e: + print(f"[WARN] LEFT JOIN failed: {e}") + + # ============================================================== + # 15. JOIN + GROUP BY + aggregates + # ============================================================== + heading(15, "SQL -- JOIN + GROUP BY + Aggregates") + sql = ( + f"SELECT t.new_code, COUNT(tk.new_sqldemotaskid) as task_count, " + f"SUM(tk.new_hours) as total_hours " + f"FROM {parent_table} t " + f"JOIN {child_table} tk ON t.{parent_id_col} = tk.{lookup_col} " + f"GROUP BY t.new_code" + ) + log_call('client.query.sql("...JOIN...GROUP BY...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} groups:") + for r in results: + print(f" Team={r.get('new_code', ''):<10s} Tasks={r.get('task_count')} Hours={r.get('total_hours')}") + except Exception as e: + print(f"[WARN] JOIN+GROUP BY failed: {e}") + + # ============================================================== + # 16. OFFSET FETCH (server-side pagination) + # ============================================================== + heading(16, "SQL -- OFFSET FETCH (Server-Side Pagination)") + page_size = 3 + for pg in range(1, 4): + offset = (pg - 1) * page_size + sql = ( + f"SELECT new_title, new_hours FROM {child_table} " + f"ORDER BY new_hours " + f"OFFSET {offset} ROWS FETCH NEXT {page_size} ROWS ONLY" + ) + log_call(f"Page {pg}: OFFSET {offset} FETCH NEXT {page_size}") + results = await backoff(lambda sql=sql: client.query.sql(sql)) + print(f" Page {pg}: {len(results)} rows") + for r in results: + print(f" {r.get('new_title', ''):<25s} Hours={r.get('new_hours')}") + if len(results) < page_size: + break + + # ============================================================== + # 17. SQL to DataFrame + # ============================================================== + heading(17, "SQL to DataFrame (client.dataframe.sql)") + print("Get SQL results directly as a pandas DataFrame.") + sql = f"SELECT new_code, new_budget, new_region " f"FROM {parent_table} ORDER BY new_budget DESC" + log_call(f'client.dataframe.sql("{sql}")') + df = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] DataFrame: {len(df)} rows x {len(df.columns)} columns") + print(df.to_string(index=False)) + print(f"\n Mean budget: {df['new_budget'].mean():,.2f}") + print(f" Budget by region:\n{df.groupby('new_region')['new_budget'].sum()}") + + # ============================================================== + # 18. SQL to DataFrame with JOINs + # ============================================================== + heading(18, "SQL to DataFrame -- JOIN Query") + sql = ( + f"SELECT t.new_code, tk.new_title, tk.new_hours " + f"FROM {child_table} tk " + f"JOIN {parent_table} t ON tk.{lookup_col} = t.{parent_id_col}" + ) + log_call('client.dataframe.sql("...JOIN...")') + try: + df_j = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] {len(df_j)} rows") + print(df_j.to_string(index=False)) + print("\n-- Pivot: hours by team --") + print(df_j.groupby("new_code")["new_hours"].agg(["sum", "mean", "count"]).to_string()) + except Exception as e: + print(f"[WARN] {e}") + + # ============================================================== + # 19. Built-in table JOINs + # ============================================================== + heading(19, "Built-In Table JOINs (account -> contact)") + sql = "SELECT a.name, c.fullname FROM account a " "INNER JOIN contact c ON a.accountid = c.parentcustomerid" + log_call('client.query.sql("...account JOIN contact...")') + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] {len(results)} rows") + for r in results[:5]: + print(f" Account={r.get('name', ''):<25s} Contact={r.get('fullname', '')}") + except Exception as e: + print(f"[INFO] {e}") + + # ============================================================== + # 20. LIMITATION: Writes require SDK + # ============================================================== + heading(20, "LIMITATION: Writes Require SDK (Read-Only SQL)") + sql = f"SELECT new_sqldemotaskid, new_title " f"FROM {child_table} WHERE new_done = 0" + incomplete = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] SQL found {len(incomplete)} incomplete tasks") + if incomplete: + fid = incomplete[0].get("new_sqldemotaskid") + if fid: + await backoff(lambda: client.records.update(child_table, fid, {"new_Done": True})) + print(f"[OK] Updated via SDK: '{incomplete[0].get('new_title')}'") + + # ============================================================== + # 21. LIMITATION: No subqueries + # ============================================================== + heading(21, "LIMITATION: No Subqueries -- Chain SQL Calls") + sql1 = f"SELECT {parent_id_col} FROM {parent_table} WHERE new_budget > 50000" + big = await backoff(lambda: client.query.sql(sql1)) + big_ids = [r.get(parent_id_col) for r in big if r.get(parent_id_col)] + print(f"[OK] Step 1: {len(big_ids)} teams with budget > 50000") + if big_ids: + id_list = ", ".join(f"'{i}'" for i in big_ids) + sql2 = f"SELECT new_title FROM {child_table} " f"WHERE {lookup_col} IN ({id_list})" + tasks_r = await backoff(lambda: client.query.sql(sql2)) + print(f"[OK] Step 2: {len(tasks_r)} tasks for big-budget teams") + + # ============================================================== + # 22. LIMITATION: No functions + # ============================================================== + heading(22, "LIMITATION: No Functions -- Post-Process in Python") + sql = f"SELECT new_code, new_budget FROM {parent_table}" + rows = await backoff(lambda: client.query.sql(sql)) + print("[OK] Post-processing (CASE equivalent):") + for r in rows: + b = float(r.get("new_budget") or 0) + tier = "HIGH" if b > 60000 else "MEDIUM" if b > 35000 else "LOW" + print(f" {r.get('new_code', ''):<12s} Budget={b:>10,.2f} Tier={tier}") + + # ============================================================== + # 23. Polymorphic lookups via SQL (ownerid, customerid) + # ============================================================== + heading(23, "Polymorphic Lookups via SQL (ownerid, customerid)") + print( + "Some Dataverse lookup columns are POLYMORPHIC -- the GUID can\n" + "point to different entity types (e.g. ownerid -> systemuser OR\n" + "team, customerid -> account OR contact).\n" + "\n" + "SQL pattern: INNER JOIN acts as both a join AND a type filter.\n" + "If the GUID points to a different type, the JOIN simply returns\n" + "no row -- so you get exactly the records of the type you joined." + ) + + # 23a. Discover lookup columns on a table + print("\n-- 23a. Discover lookup columns on account --") + log_call("client.tables.list_columns('account', filter=Lookup)") + try: + acct_cols = await backoff( + lambda: client.tables.list_columns( + "account", + select=["LogicalName", "AttributeType"], + filter="AttributeType eq 'Lookup' or AttributeType eq 'Owner' or AttributeType eq 'Customer'", + ) + ) + lookup_names = sorted(c.get("LogicalName", "") for c in acct_cols if c.get("LogicalName", "")) + print(f"[OK] Lookup columns on account ({len(lookup_names)} found):") + for ln in lookup_names[:10]: + print(f" {ln}") + if len(lookup_names) > 10: + print(f" ... and {len(lookup_names) - 10} more") + except Exception as e: + print(f"[INFO] Lookup discovery skipped: {e}") + + # 23b. Discover polymorphic targets via relationship metadata + print("\n-- 23b. Discover which entities a polymorphic lookup targets --") + log_call("client.tables.list_table_relationships('account', ...)") + try: + acct_rels = await backoff(lambda: client.tables.list_table_relationships("account")) + by_attr = defaultdict(list) + for rel in acct_rels: + attr = rel.get("ReferencingAttribute", "") + ref = rel.get("ReferencedEntity", "") + if attr and ref and rel.get("ReferencingEntity", "").lower() == "account": + by_attr[attr].append(ref) + print("[OK] Lookup targets on account:") + for attr, targets in sorted(by_attr.items()): + tag = "POLYMORPHIC" if len(targets) > 1 else "regular" + print(f" {attr:<35s} -> {', '.join(targets):<30s} [{tag}]") + except Exception as e: + print(f"[INFO] Relationship discovery skipped: {e}") + + # 23c. Resolve ownerid (polymorphic: systemuser or team) + print("\n-- 23c. Resolve ownerid via SQL JOINs --") + print("ownerid is polymorphic (systemuser or team). Use separate\n" "JOINs and combine in a DataFrame.") + try: + # Records owned by users + log_call("SQL: account JOIN systemuser ON ownerid") + df_user_owned = await backoff( + lambda: client.dataframe.sql( + "SELECT TOP 5 a.name, su.fullname as owner_name " + "FROM account a " + "INNER JOIN systemuser su ON a.ownerid = su.systemuserid" + ) + ) + df_user_owned["owner_type"] = "User" + + # Records owned by teams + log_call("SQL: account JOIN team ON ownerid") + df_team_owned = await backoff( + lambda: client.dataframe.sql( + "SELECT TOP 5 a.name, t.name as owner_name " + "FROM account a " + "INNER JOIN team t ON a.ownerid = t.teamid" + ) + ) + df_team_owned["owner_type"] = "Team" + + df_owners = pd.concat([df_user_owned, df_team_owned], ignore_index=True) + print(f"[OK] Owner resolution: {len(df_owners)} rows") + print(f" User-owned: {len(df_user_owned)}") + print(f" Team-owned: {len(df_team_owned)}") + if not df_owners.empty: + print(df_owners.to_string(index=False)) + except Exception as e: + print(f"[INFO] Owner resolution skipped (may have no data): {e}") + + # 23d. Track created-by and modified-by (common audit pattern) + print("\n-- 23d. Audit trail: who created/modified records (via SQL) --") + try: + log_call("SQL: account JOIN systemuser (createdby + modifiedby)") + results = await backoff( + lambda: client.query.sql( + "SELECT TOP 5 a.name, " + "creator.fullname as created_by, " + "modifier.fullname as modified_by " + "FROM account a " + "JOIN systemuser creator ON a.createdby = creator.systemuserid " + "JOIN systemuser modifier ON a.modifiedby = modifier.systemuserid" + ) + ) + print(f"[OK] Audit trail: {len(results)} rows") + for r in results[:5]: + print( + f" {r.get('name', ''):<25s} " + f"Created: {r.get('created_by', ''):<20s} " + f"Modified: {r.get('modified_by', '')}" + ) + except Exception as e: + print(f"[INFO] Audit trail skipped: {e}") + + # ============================================================== + # 24. SQL Read -> DataFrame Transform -> SDK Write-Back + # ============================================================== + heading(24, "SQL Read -> DataFrame Transform -> SDK Write-Back") + print( + "The full bidirectional workflow for SQL users:\n" + " 1. SQL query -> DataFrame (read)\n" + " 2. pandas -> Transform (compute)\n" + " 3. DataFrame -> SDK write-back (create/update/delete)\n" + "\n" + "This is how SQL developers do end-to-end work without\n" + "learning OData or the Web API." + ) + + # Read current state via SQL + sql = f"SELECT new_sqldemotaskid, new_title, new_hours, new_done " f"FROM {child_table}" + log_call(f'client.dataframe.sql("{sql}")') + df_tasks = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] Read {len(df_tasks)} tasks via SQL") + print(df_tasks.to_string(index=False)) + + # Transform: bump hours by 1 for incomplete tasks + mask = df_tasks["new_done"] == False # noqa: E712 + original_hours = df_tasks.loc[mask, "new_hours"].copy() + df_tasks.loc[mask, "new_hours"] = df_tasks.loc[mask, "new_hours"] + 1 + changed = mask.sum() + print(f"\n[OK] Bumped hours +1 for {changed} incomplete tasks (in DataFrame)") + + # Write back via SDK + if changed > 0: + updates = df_tasks.loc[mask, ["new_sqldemotaskid", "new_hours"]] + log_call(f"client.dataframe.update('{child_table}', ..., id_column='new_sqldemotaskid')") + await backoff(lambda: client.dataframe.update(child_table, updates, id_column="new_sqldemotaskid")) + print(f"[OK] Wrote back {len(updates)} updated rows via DataFrame") + + # Verify with SQL + verify = await backoff( + lambda: client.dataframe.sql(f"SELECT new_title, new_hours FROM {child_table} WHERE new_done = 0") + ) + print(f"[OK] Verified via SQL -- incomplete tasks now:") + print(verify.to_string(index=False)) + + # Restore original values + df_tasks.loc[mask, "new_hours"] = original_hours + restore = df_tasks.loc[mask, ["new_sqldemotaskid", "new_hours"]] + await backoff(lambda: client.dataframe.update(child_table, restore, id_column="new_sqldemotaskid")) + print("[OK] Restored original hours") + + # ============================================================== + # 25. SQL-driven bulk create from query results + # ============================================================== + heading(25, "SQL-Driven Bulk Create (Query -> Transform -> Insert)") + print( + "Pattern: query existing data with SQL, transform it,\n" + "then create new records via DataFrame -- all without\n" + "learning OData syntax." + ) + + # Read teams via SQL + sql = f"SELECT new_code, new_budget FROM {parent_table} WHERE new_active = 1" + log_call(f'client.dataframe.sql("{sql}")') + df_active = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] Read {len(df_active)} active teams via SQL") + + # Transform: create a new task for each active team + new_tasks = pd.DataFrame( + { + "new_Title": [f"Review budget for {code}" for code in df_active["new_code"]], + "new_Hours": [2] * len(df_active), + "new_Done": [False] * len(df_active), + "new_Priority": [1] * len(df_active), + } + ) + log_call(f"client.dataframe.create('{child_table}', DataFrame({len(new_tasks)} rows))") + new_ids = await backoff(lambda: client.dataframe.create(child_table, new_tasks)) + print(f"[OK] Created {len(new_ids)} new tasks from SQL query results") + + # Verify with SQL + verify_sql = f"SELECT new_title, new_hours FROM {child_table} " f"WHERE new_title LIKE 'Review budget%'" + created_tasks = await backoff(lambda: client.query.sql(verify_sql)) + print(f"[OK] Verified via SQL: {len(created_tasks)} 'Review budget' tasks") + + # Clean up the created tasks + await backoff(lambda: client.dataframe.delete(child_table, new_ids)) + print(f"[OK] Cleaned up {len(new_ids)} demo tasks") + + # ============================================================== + # 26. SQL-driven bulk delete + # ============================================================== + heading(26, "SQL-Driven Bulk Delete (Query -> Filter -> Delete)") + print("Pattern: find records with SQL, filter in pandas,\n" "then delete via DataFrame -- pure SQL thinking.") + + # Create some temp records to demonstrate + temp = pd.DataFrame( + { + "new_Title": ["TEMP: delete me 1", "TEMP: delete me 2", "TEMP: keep me"], + "new_Hours": [1, 2, 3], + "new_Done": [False, False, False], + "new_Priority": [1, 1, 1], + } + ) + temp_ids = await backoff(lambda: client.dataframe.create(child_table, temp)) + print(f"[OK] Created {len(temp_ids)} temp records") + + # SQL to find, pandas to filter, SDK to delete + sql = f"SELECT new_sqldemotaskid, new_title FROM {child_table} WHERE new_title LIKE 'TEMP:%'" + df_temp = await backoff(lambda: client.dataframe.sql(sql)) + print(f"[OK] SQL found {len(df_temp)} TEMP records") + + # Filter in pandas: only delete the "delete me" ones + to_delete = df_temp[df_temp["new_title"].str.contains("delete me")] + print(f"[OK] Pandas filtered to {len(to_delete)} records to delete") + + if not to_delete.empty: + log_call("client.dataframe.delete(...)") + await backoff(lambda: client.dataframe.delete(child_table, to_delete["new_sqldemotaskid"])) + print(f"[OK] Deleted {len(to_delete)} records via DataFrame") + + # Verify the "keep me" record survived + remaining = await backoff( + lambda: client.query.sql(f"SELECT new_title FROM {child_table} WHERE new_title LIKE 'TEMP:%'") + ) + print(f"[OK] Remaining TEMP records: {len(remaining)}") + for r in remaining: + print(f" {r.get('new_title')}") + + # Clean up the surviving temp record + keep_ids = [ + r.get("new_sqldemotaskid") + for r in await backoff( + lambda: client.query.sql(f"SELECT new_sqldemotaskid FROM {child_table} WHERE new_title LIKE 'TEMP:%'") + ) + if r.get("new_sqldemotaskid") + ] + for kid in keep_ids: + await backoff(lambda kid=kid: client.records.delete(child_table, kid)) + + # ============================================================== + # 27. AND/OR, NOT IN, NOT LIKE + # ============================================================== + heading(27, "SQL -- AND/OR, NOT IN, NOT LIKE") + sql = f"SELECT new_code, new_budget FROM {parent_table} " f"WHERE new_active = 1 AND new_budget > 40000" + log_call(f'client.query.sql("{sql}")') + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] AND: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} " f"WHERE new_code = 'ALPHA' OR new_code = 'DELTA'" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] OR: {[r.get('new_code') for r in results]}") + + sql = ( + f"SELECT new_code FROM {parent_table} " + f"WHERE new_active = 1 AND (new_budget > 80000 OR new_budget < 45000)" + ) + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] AND + OR with parens: {len(results)} rows") + + sql = f"SELECT new_code FROM {parent_table} WHERE new_code NOT IN ('ALPHA')" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] NOT IN: {[r.get('new_code') for r in results]}") + + sql = f"SELECT new_title FROM {child_table} WHERE new_title NOT LIKE 'Design%'" + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] NOT LIKE: {len(results)} rows") + + # ============================================================== + # 28. Deep JOINs (5-8 tables) + # ============================================================== + heading(28, "Deep JOINs (5+ Tables) -- No Depth Limit") + + sql = ( + "SELECT TOP 3 a.name, c.fullname, o.name as opp, " + "su.fullname as owner, bu.name as bu " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "JOIN opportunity o ON a.accountid = o.parentaccountid " + "JOIN systemuser su ON a.ownerid = su.systemuserid " + "JOIN businessunit bu ON su.businessunitid = bu.businessunitid" + ) + log_call("5-table JOIN") + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] 5-table JOIN: {len(results)} rows") + except Exception as e: + print(f"[INFO] {e}") + + # ============================================================== + # 29. SQL Helper Functions + # ============================================================== + heading(29, "SQL Helper Functions (query.sql_*)") + print( + "At GA, sql_columns() is the only retained SQL schema-discovery helper.\n" + "sql_select(), sql_join(), and sql_joins() were removed -- write JOIN\n" + "clauses directly or use client.query.fetchxml() for complex queries." + ) + + # sql_columns — still available at GA + log_call(f"client.query.sql_columns('{parent_table}')") + cols = await client.query.sql_columns(parent_table) + print(f"[OK] {len(cols)} columns:") + for c in cols[:5]: + print(f" {c['name']:30s} Type: {c['type']:15s} PK={c['is_pk']}") + + # ============================================================== + # 30. OData Helper Functions + # ============================================================== + heading(30, "OData Helper Functions (query.odata_expands)") + print( + "odata_expands() is available without deprecation.\n" + "odata_select(), odata_expand(), and odata_bind() are deprecated\n" + "at GA -- use the typed query builder instead." + ) + + # odata_expands + log_call(f"client.query.odata_expands('{child_table}')") + try: + expands = await client.query.odata_expands(child_table) + print(f"[OK] {len(expands)} expand targets:") + for e in expands[:5]: + print(f" nav={e['nav_property']:30s} -> {e['target_table']}") + except Exception as e: + print(f"[WARN] {e}") + + finally: + heading(31, "Cleanup") + for tbl in [child_table, parent_table]: + log_call(f"client.tables.delete('{tbl}')") + try: + await backoff(lambda tbl=tbl: client.tables.delete(tbl)) + print(f"[OK] Deleted table: {tbl}") + except Exception as ex: + code = getattr(getattr(ex, "response", None), "status_code", None) + if isinstance(ex, MetadataError) and code == 404: + print(f"[OK] Table already removed: {tbl}") + else: + print(f"[WARN] Could not delete {tbl}: {ex}") + + print("\n" + "=" * 80) + print("Async SQL Examples Complete!") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/advanced/walkthrough.py b/examples/aio/advanced/walkthrough.py new file mode 100644 index 00000000..d7a14e4a --- /dev/null +++ b/examples/aio/advanced/walkthrough.py @@ -0,0 +1,632 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async walkthrough demonstrating core Dataverse SDK operations. + +Async equivalent of examples/advanced/walkthrough.py. + +This example shows: +- Table creation with various column types including enums +- Single and multiple record CRUD operations +- Querying with filtering, paging, AsyncQueryBuilder, and SQL +- Expand (navigation properties) with AsyncQueryBuilder +- Picklist label-to-value conversion +- Column management +- Batch operations (create, read, update, changeset, delete in one HTTP request) +- Cleanup + +Prerequisites: +- pip install PowerPlatform-Dataverse-Client +- pip install azure-identity +""" + +import asyncio +import sys +import json +from enum import IntEnum + +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import MetadataError +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + + +def log_call(description): + print(f"\n-> {description}") + + +class Priority(IntEnum): + LOW = 1 + MEDIUM = 2 + HIGH = 3 + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off for metadata propagation delays.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" [INFO] Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print(f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total.") + raise last + + +async def main(): + print("=" * 80) + print("Dataverse SDK Async Walkthrough") + print("=" * 80) + + # ============================================================================ + # 1. SETUP & AUTHENTICATION + # ============================================================================ + print("\n" + "=" * 80) + print("1. Setup & Authentication") + print("=" * 80) + + base_url = input("Enter Dataverse org URL (e.g. https://yourorg.crm.dynamics.com): ").strip() + if not base_url: + print("No URL entered; exiting.") + sys.exit(1) + + base_url = base_url.rstrip("/") + + log_call("AsyncInteractiveBrowserCredential()") + credential = AsyncInteractiveBrowserCredential() + + log_call(f"AsyncDataverseClient(base_url='{base_url}', credential=...)") + try: + async with AsyncDataverseClient(base_url=base_url, credential=credential) as client: + print(f"[OK] Connected to: {base_url}") + await _run_walkthrough(client) + finally: + await credential.close() + + +async def _run_walkthrough(client): + # ============================================================================ + # 2. TABLE CREATION (METADATA) + # ============================================================================ + print("\n" + "=" * 80) + print("2. Table Creation (Metadata)") + print("=" * 80) + + table_name = "new_WalkthroughDemo" + + log_call(f"await client.tables.get('{table_name}')") + table_info = await backoff(lambda: client.tables.get(table_name)) + + if table_info: + print(f"[OK] Table already exists: {table_info.get('table_schema_name')}") + print(f" Logical Name: {table_info.get('table_logical_name')}") + print(f" Entity Set: {table_info.get('entity_set_name')}") + else: + log_call(f"await client.tables.create('{table_name}', columns={{...}}, display_name='Walkthrough Demo')") + columns = { + "new_Title": "string", + "new_Quantity": "int", + "new_Amount": "decimal", + "new_Completed": "bool", + "new_Notes": "memo", + "new_Priority": Priority, + } + table_info = await backoff(lambda: client.tables.create(table_name, columns, display_name="Walkthrough Demo")) + print(f"[OK] Created table: {table_info.get('table_schema_name')}") + print(f" Columns created: {', '.join(table_info.get('columns_created', []))}") + + # ============================================================================ + # 3. CREATE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("3. Create Operations") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', {{...}})") + single_record = { + "new_Title": "Complete project documentation", + "new_Quantity": 5, + "new_Amount": 1250.50, + "new_Completed": False, + "new_Notes": "This is a multiline memo field.\nIt supports longer text content.", + "new_Priority": Priority.MEDIUM, + } + id1 = await backoff(lambda: client.records.create(table_name, single_record)) + print(f"[OK] Created single record: {id1}") + + log_call(f"await client.records.create('{table_name}', [{{...}}, {{...}}, {{...}}])") + multiple_records = [ + { + "new_Title": "Review code changes", + "new_Quantity": 10, + "new_Amount": 500.00, + "new_Completed": True, + "new_Priority": Priority.HIGH, + }, + { + "new_Title": "Update test cases", + "new_Quantity": 8, + "new_Amount": 750.25, + "new_Completed": False, + "new_Priority": Priority.LOW, + }, + { + "new_Title": "Deploy to staging", + "new_Quantity": 3, + "new_Amount": 2000.00, + "new_Completed": False, + "new_Priority": Priority.HIGH, + }, + ] + ids = await backoff(lambda: client.records.create(table_name, multiple_records)) + print(f"[OK] Created {len(ids)} records: {ids}") + + # ============================================================================ + # 4. READ OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("4. Read Operations") + print("=" * 80) + + log_call(f"await client.records.retrieve('{table_name}', '{id1}')") + record = await backoff(lambda: client.records.retrieve(table_name, id1)) + print("[OK] Retrieved single record:") + print( + json.dumps( + { + "new_walkthroughdemoid": record.get("new_walkthroughdemoid"), + "new_title": record.get("new_title"), + "new_quantity": record.get("new_quantity"), + "new_amount": record.get("new_amount"), + "new_completed": record.get("new_completed"), + "new_notes": record.get("new_notes"), + "new_priority": record.get("new_priority"), + "new_priority@FormattedValue": record.get("new_priority@OData.Community.Display.V1.FormattedValue"), + }, + indent=2, + ) + ) + + log_call(f"await client.records.list('{table_name}', filter='new_quantity gt 5')") + all_records = await backoff(lambda: client.records.list(table_name, filter="new_quantity gt 5")) + print(f"[OK] Found {len(all_records)} records with new_quantity > 5") + for rec in all_records: + print(f" - new_Title='{rec.get('new_title')}', new_Quantity={rec.get('new_quantity')}") + + # ============================================================================ + # 5. UPDATE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("5. Update Operations") + print("=" * 80) + + log_call(f"await client.records.update('{table_name}', '{id1}', {{...}})") + await backoff( + lambda: client.records.update( + table_name, + id1, + { + "new_Quantity": 100, + "new_Notes": "Updated memo field.\nNow with revised content across multiple lines.", + }, + ) + ) + updated = await backoff(lambda: client.records.retrieve(table_name, id1)) + print(f"[OK] Updated single record new_Quantity: {updated.get('new_quantity')}") + print(f" new_Notes: {repr(updated.get('new_notes'))}") + + log_call(f"await client.records.update('{table_name}', [{len(ids)} IDs], {{...}})") + await backoff(lambda: client.records.update(table_name, ids, {"new_Completed": True})) + print(f"[OK] Updated {len(ids)} records to new_Completed=True") + + # ============================================================================ + # 6. PAGING DEMO + # ============================================================================ + print("\n" + "=" * 80) + print("6. Paging Demo") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', [20 records])") + paging_records = [ + { + "new_Title": f"Paging test item {i}", + "new_Quantity": i, + "new_Amount": i * 10.0, + "new_Completed": False, + "new_Priority": Priority.LOW, + } + for i in range(1, 21) + ] + paging_ids = await backoff(lambda: client.records.create(table_name, paging_records)) + print(f"[OK] Created {len(paging_ids)} records for paging demo") + + log_call(f"async for page in client.query.builder('{table_name}').order_by().page_size(5).execute_pages()") + print("Fetching records with page_size=5...") + page_num = 0 + async for page in client.query.builder(table_name).order_by("new_Quantity").page_size(5).execute_pages(): + page_num += 1 + record_ids = [r.get("new_walkthroughdemoid")[:8] + "..." for r in page] + print(f" Page {page_num}: {len(page)} records - IDs: {record_ids}") + + # ============================================================================ + # 7. QUERYBUILDER - FLUENT QUERIES + # ============================================================================ + print("\n" + "=" * 80) + print("7. AsyncQueryBuilder - Fluent Queries") + print("=" * 80) + + log_call("await client.query.builder(...).select().where(col(...)==...).order_by().execute()") + print("Querying incomplete records ordered by amount (fluent builder)...") + qb_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Priority") + .where(col("new_Completed") == False) + .order_by("new_Amount", descending=True) + .top(10) + .execute() + ) + print(f"[OK] AsyncQueryBuilder found {len(qb_result)} incomplete records:") + for rec in list(qb_result)[:5]: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')}") + + log_call("await client.query.builder(...).where(col('new_Priority').in_([HIGH, LOW])).execute()") + print("Querying records with HIGH or LOW priority (col().in_())...") + priority_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Priority") + .where(col("new_Priority").in_([Priority.HIGH, Priority.LOW])) + .execute() + ) + print(f"[OK] Found {len(priority_result)} records with HIGH or LOW priority") + + log_call("await client.query.builder(...).where(col('new_Amount').between(500, 1500)).execute()") + range_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount") + .where(col("new_Amount").between(500, 1500)) + .execute() + ) + print(f"[OK] Found {len(range_result)} records with amount in [500, 1500]") + + log_call("await client.query.builder(...).where((col(...)==...) & (col(...) > ...)).execute()") + expr_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_Title", "new_Amount", "new_Quantity") + .where((col("new_Completed") == False) & (col("new_Amount") > 100)) + .order_by("new_Amount", descending=True) + .top(5) + .execute() + ) + print(f"[OK] Expression tree query found {len(expr_result)} records:") + for rec in expr_result: + print(f" - '{rec.get('new_title')}' Amount={rec.get('new_amount')} Qty={rec.get('new_quantity')}") + + log_call("async for page in client.query.builder(...).where(...).page_size().execute_pages()") + print("Querying with combined expression filters and paging...") + combined_page_count = 0 + combined_record_count = 0 + async for page in ( + client.query.builder(table_name) + .select("new_Title", "new_Quantity") + .where(col("new_Completed") == False) + .where(col("new_Quantity").between(1, 15)) + .order_by("new_Quantity") + .page_size(3) + .execute_pages() + ): + combined_page_count += 1 + combined_record_count += len(page) + titles = [r.get("new_title", "?") for r in page] + print(f" Page {combined_page_count}: {len(page)} records - {titles}") + print(f"[OK] Combined query: {combined_record_count} records across {combined_page_count} page(s)") + + log_call(f"(await client.query.builder('{table_name}').select(...).where(...).execute()).to_dataframe()") + print("Querying completed records as a pandas DataFrame (to_dataframe)...") + completed_result = await backoff( + lambda: client.query.builder(table_name) + .select("new_title", "new_quantity") + .where(col("new_completed") == True) + .execute() + ) + df = completed_result.to_dataframe() + print(f"[OK] to_dataframe() returned {len(df)} rows, columns: {list(df.columns)}") + if not df.empty: + print(f" First row: new_title='{df.iloc[0].get('new_title')}', new_quantity={df.iloc[0].get('new_quantity')}") + print(f" Sum of new_quantity: {df['new_quantity'].sum()}") + else: + print(" (empty DataFrame)") + + # ============================================================================ + # 8. EXPAND (NAVIGATION PROPERTIES) + # ============================================================================ + print("\n" + "=" * 80) + print("8. Expand (Navigation Properties)") + print("=" * 80) + + log_call("await client.query.builder('account').select('name').expand('primarycontactid').top(3).execute()") + try: + expanded_records = await backoff( + lambda: client.query.builder("account").select("name").expand("primarycontactid").top(3).execute() + ) + print(f"[OK] Found {len(expanded_records)} accounts with expanded contact:") + for rec in expanded_records: + contact = rec.get("primarycontactid") + contact_name = contact.get("fullname", "(none)") if contact else "(no contact)" + print(f" - '{rec.get('name')}' -> Contact: {contact_name}") + except Exception as e: + print(f"[SKIP] Expand demo skipped (no accounts in org): {e}") + + log_call("ExpandOption('Account_Tasks').select('subject').order_by('createdon', descending=True).top(3)") + try: + tasks_opt = ( + ExpandOption("Account_Tasks").select("subject", "createdon").order_by("createdon", descending=True).top(3) + ) + nested_records = await backoff( + lambda: client.query.builder("account").select("name").expand(tasks_opt).top(3).execute() + ) + print(f"[OK] Found {len(nested_records)} accounts with nested task expansion:") + for rec in nested_records: + tasks = rec.get("Account_Tasks", []) + print(f" - '{rec.get('name')}' has {len(tasks)} task(s)") + except Exception as e: + print(f"[SKIP] Nested expand demo skipped: {e}") + + # ============================================================================ + # 9. SQL QUERY + # ============================================================================ + print("\n" + "=" * 80) + print("9. SQL Query") + print("=" * 80) + + sql = "SELECT new_title, new_quantity FROM new_walkthroughdemo WHERE new_completed = 1" + log_call(f"await client.query.sql('{sql}')") + try: + results = await backoff(lambda: client.query.sql(sql)) + print(f"[OK] SQL query returned {len(results)} completed records:") + for result in results[:5]: + print(f" - new_Title='{result.get('new_title')}', new_Quantity={result.get('new_quantity')}") + except Exception as e: + print(f"[WARN] SQL query failed: {str(e)}") + + # ============================================================================ + # 10. FETCHXML QUERY + # ============================================================================ + print("\n" + "=" * 80) + print("10. FetchXML Query") + print("=" * 80) + + xml = f""" + + + + + + + + + + """ + log_call("await client.query.fetchxml(xml).execute()") + try: + fx_result = await backoff(lambda: client.query.fetchxml(xml).execute()) + print(f"[OK] FetchXML returned {len(fx_result)} incomplete records:") + for r in fx_result[:5]: + print(f" - '{r.get('new_title')}' Quantity={r.get('new_quantity')}") + except Exception as e: + print(f"[WARN] FetchXML query failed: {e}") + + log_call("async for page in client.query.fetchxml(paged_xml).execute_pages()") + paged_xml = f""" + + + + + + + """ + try: + fx_page_num = 0 + fx_total = 0 + async for page in client.query.fetchxml(paged_xml).execute_pages(): + fx_page_num += 1 + fx_total += len(page) + titles = [r.get("new_title", "?") for r in page] + print(f" Page {fx_page_num}: {len(page)} record(s) — {titles}") + print(f"[OK] FetchXML execute_pages(): {fx_total} total records across {fx_page_num} page(s)") + except Exception as e: + print(f"[WARN] FetchXML execute_pages failed: {e}") + + # ============================================================================ + # 11. PICKLIST LABEL CONVERSION + # ============================================================================ + print("\n" + "=" * 80) + print("11. Picklist Label Conversion") + print("=" * 80) + + log_call(f"await client.records.create('{table_name}', {{'new_Priority': 'High'}})") + label_record = { + "new_Title": "Test label conversion", + "new_Quantity": 1, + "new_Amount": 99.99, + "new_Completed": False, + "new_Priority": "High", + } + label_id = await backoff(lambda: client.records.create(table_name, label_record)) + retrieved = await backoff(lambda: client.records.retrieve(table_name, label_id)) + print(f"[OK] Created record with string label 'High' for new_Priority") + print(f" new_Priority stored as integer: {retrieved.get('new_priority')}") + print(f" new_Priority@FormattedValue: {retrieved.get('new_priority@OData.Community.Display.V1.FormattedValue')}") + + log_call(f"await client.records.update('{table_name}', label_id, {{'new_Priority': 'Low'}})") + await backoff(lambda: client.records.update(table_name, label_id, {"new_Priority": "Low"})) + updated_label = await backoff(lambda: client.records.retrieve(table_name, label_id)) + print(f"[OK] Updated record with string label 'Low' for new_Priority") + print(f" new_Priority stored as integer: {updated_label.get('new_priority')}") + + # ============================================================================ + # 12. COLUMN MANAGEMENT + # ============================================================================ + print("\n" + "=" * 80) + print("12. Column Management") + print("=" * 80) + + log_call(f"await client.tables.add_columns('{table_name}', {{'new_Tags': 'string'}})") + created_cols = await backoff(lambda: client.tables.add_columns(table_name, {"new_Tags": "string"})) + print(f"[OK] Added column: {created_cols[0]}") + + log_call(f"await client.tables.remove_columns('{table_name}', ['new_Tags'])") + await backoff(lambda: client.tables.remove_columns(table_name, ["new_Tags"])) + print("[OK] Deleted column: new_Tags") + + # ============================================================================ + # 13. DELETE OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("13. Delete Operations") + print("=" * 80) + + log_call(f"await client.records.delete('{table_name}', '{id1}')") + await backoff(lambda: client.records.delete(table_name, id1)) + print(f"[OK] Deleted single record: {id1}") + + log_call(f"await client.records.delete('{table_name}', [{len(paging_ids)} IDs])") + job_id = await backoff(lambda: client.records.delete(table_name, paging_ids)) + print(f"[OK] Bulk delete job started: {job_id}") + + # ============================================================================ + # 14. BATCH OPERATIONS + # ============================================================================ + print("\n" + "=" * 80) + print("14. Batch Operations") + print("=" * 80) + + log_call("client.batch.new() + batch.records.create(...) x2 + await batch.execute()") + batch = client.batch.new() + batch.records.create( + table_name, + { + "new_Title": "Batch task alpha", + "new_Quantity": 1, + "new_Amount": 25.0, + "new_Completed": False, + "new_Priority": Priority.LOW, + }, + ) + batch.records.create( + table_name, + { + "new_Title": "Batch task beta", + "new_Quantity": 2, + "new_Amount": 50.0, + "new_Completed": False, + "new_Priority": Priority.MEDIUM, + }, + ) + result = await batch.execute() + batch_ids = list(result.entity_ids) + print(f"[OK] Batch create: {len(result.succeeded)} operations, {len(batch_ids)} records created") + + log_call("client.batch.new() + batch.records.retrieve(...) x2 + await batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.retrieve(table_name, bid, select=["new_title", "new_quantity"]) + result = await batch.execute() + print(f"[OK] Batch get: {len(result.succeeded)} reads in one HTTP request") + for resp in result.succeeded: + if resp.data: + print(f" new_title='{resp.data.get('new_title')}', new_quantity={resp.data.get('new_quantity')}") + + log_call("async with batch.changeset() as cs: cs.records.create(...); cs.records.update(ref, ...)") + batch = client.batch.new() + async with batch.changeset() as cs: + cs_ref = cs.records.create( + table_name, + { + "new_Title": "Changeset task", + "new_Quantity": 5, + "new_Amount": 100.0, + "new_Completed": False, + "new_Priority": Priority.HIGH, + }, + ) + cs.records.update(table_name, cs_ref, {"new_Completed": True}) + result = await batch.execute() + if not result.has_errors: + batch_ids.extend(result.entity_ids) + print(f"[OK] Changeset: {len(result.succeeded)} operations committed atomically") + else: + for item in result.failed: + print(f"[WARN] Changeset error {item.status_code}: {item.error_message}") + + log_call(f"client.batch.new() + batch.records.delete(...) x{len(batch_ids)} + await batch.execute()") + batch = client.batch.new() + for bid in batch_ids: + batch.records.delete(table_name, bid) + result = await batch.execute(continue_on_error=True) + print(f"[OK] Batch delete: {len(result.succeeded)} records deleted in one HTTP request") + + # ============================================================================ + # 15. CLEANUP + # ============================================================================ + print("\n" + "=" * 80) + print("15. Cleanup") + print("=" * 80) + + log_call(f"await client.tables.delete('{table_name}')") + try: + await backoff(lambda: client.tables.delete(table_name)) + print(f"[OK] Deleted table: {table_name}") + except MetadataError as ex: + if "not found" in str(ex).lower(): + print(f"[OK] Table already removed: {table_name}") + else: + raise + except Exception as ex: + if "404" in str(ex): + print(f"[OK] Table removed: {table_name}") + else: + raise + + # ============================================================================ + # SUMMARY + # ============================================================================ + print("\n" + "=" * 80) + print("Async Walkthrough Complete!") + print("=" * 80) + print("\nDemonstrated operations:") + print(" [OK] Table creation with multiple column types") + print(" [OK] Single and multiple record creation") + print(" [OK] Reading records by ID and with filters") + print(" [OK] Single and multiple record updates") + print(" [OK] Paging through large result sets") + print(" [OK] AsyncQueryBuilder fluent queries (where + col(), col().in_(), col().between(), to_dataframe)") + print(" [OK] Expand navigation properties (simple + nested ExpandOption)") + print(" [OK] SQL queries") + print(" [OK] FetchXML queries (execute + execute_pages)") + print(" [OK] Picklist label-to-value conversion") + print(" [OK] Column management") + print(" [OK] Single and bulk delete operations") + print(" [OK] Batch operations (create, read, changeset, delete)") + print(" [OK] Table cleanup") + print("=" * 80) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/basic/__init__.py b/examples/aio/basic/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/examples/aio/basic/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/examples/aio/basic/functional_testing.py b/examples/aio/basic/functional_testing.py new file mode 100644 index 00000000..b194050e --- /dev/null +++ b/examples/aio/basic/functional_testing.py @@ -0,0 +1,898 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client SDK - Async Functional Testing + +Async equivalent of examples/basic/functional_testing.py. + +This script provides comprehensive async functional testing of the SDK: +- Real environment connection testing +- Table creation and metadata operations +- Full CRUD operations testing +- Query functionality validation (list, list_pages, builder, fetchxml) +- Batch operations (create, read, update, changeset, delete) +- Interactive cleanup options + +Prerequisites: +- PowerPlatform-Dataverse-Client SDK installed (run aio/basic/installation_example.py first) +- Azure Identity credentials configured +- Access to a Dataverse environment with table creation permissions + +Usage: + python examples/aio/basic/functional_testing.py +""" + +import asyncio +import sys +from typing import Optional, Dict, Any +from datetime import datetime + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + CascadeConfiguration, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel +from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_NO_CASCADE, + CASCADE_BEHAVIOR_REMOVE_LINK, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential + + +def get_dataverse_org_url() -> str: + """Get Dataverse org URL from user input.""" + print("\n-> Dataverse Environment Setup") + print("=" * 50) + + if not sys.stdin.isatty(): + print("[ERR] Interactive input required. Run this script in a terminal.") + sys.exit(1) + + while True: + org_url = input("Enter your Dataverse org URL (e.g., https://yourorg.crm.dynamics.com): ").strip() + if org_url: + return org_url.rstrip("/") + print("[WARN] Please enter a valid URL.") + + +async def backoff(coro_fn, *, delays=(0, 2, 5, 10, 20, 20)): + """Retry a coroutine with exponential back-off for metadata propagation delays.""" + last = None + total_delay = 0 + attempts = 0 + for d in delays: + if d: + await asyncio.sleep(d) + total_delay += d + attempts += 1 + try: + result = await coro_fn() + if attempts > 1: + print(f" * Backoff succeeded after {attempts - 1} retry(s); waited {total_delay}s total.") + return result + except Exception as ex: + last = ex + continue + if last: + if attempts: + print(f" [WARN] Backoff exhausted after {max(attempts - 1, 0)} retry(s); waited {total_delay}s total.") + raise last + + +async def setup_authentication(): + """Set up authentication and create async Dataverse client.""" + print("\n-> Authentication Setup") + print("=" * 50) + + org_url = get_dataverse_org_url() + try: + credential = AsyncInteractiveBrowserCredential() + client = AsyncDataverseClient(org_url, credential) + + print("Testing connection...") + tables = await client.tables.list() + print(f"[OK] Connection successful! Found {len(tables)} tables.") + + user_owned = await client.tables.list( + filter="OwnershipType eq Microsoft.Dynamics.CRM.OwnershipTypes'UserOwned'", + select=["LogicalName", "SchemaName", "DisplayName"], + ) + print(f"[OK] Found {len(user_owned)} user-owned tables (filter + select).") + return client, credential + + except Exception as e: + print(f"[ERR] Authentication failed: {e}") + sys.exit(1) + + +async def wait_for_table_metadata( + client: AsyncDataverseClient, + table_schema_name: str, + retries: int = 10, + delay_seconds: int = 3, +) -> Dict[str, Any]: + """Poll until table metadata is published and entity set becomes available.""" + for attempt in range(1, retries + 1): + try: + info = await client.tables.get(table_schema_name) + if info and info.get("entity_set_name"): + if attempt > 1: + print(f" [OK] Table metadata available after {attempt} attempts.") + return info + except Exception: + pass + + if attempt < retries: + print(f" Waiting for table metadata to publish (attempt {attempt}/{retries})...") + await asyncio.sleep(delay_seconds) + + raise RuntimeError("Table metadata did not become available in time. Please retry later.") + + +async def ensure_test_table(client: AsyncDataverseClient) -> Dict[str, Any]: + """Create or verify test table exists.""" + print("\n-> Test Table Setup") + print("=" * 50) + + table_schema_name = "test_TestSDKFunctionality" + + try: + existing_table = await client.tables.get(table_schema_name) + if existing_table: + print(f"[OK] Test table '{table_schema_name}' already exists") + return existing_table + except Exception: + print(f"Table '{table_schema_name}' not found, creating...") + + try: + print("Creating new test table...") + table_info = await client.tables.create( + table_schema_name, + primary_column="test_name", + columns={ + "test_description": "string", + "test_count": "int", + "test_amount": "decimal", + "test_is_active": "bool", + "test_created_date": "datetime", + }, + ) + print(f"[OK] Created test table: {table_info.get('table_schema_name')}") + print(f" Logical name: {table_info.get('table_logical_name')}") + print(f" Entity set: {table_info.get('entity_set_name')}") + + return await wait_for_table_metadata(client, table_schema_name) + + except MetadataError as e: + print(f"[ERR] Failed to create table: {e}") + sys.exit(1) + + +async def test_create_record(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> str: + """Test record creation.""" + print("\n-> Record Creation Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + test_data = { + f"{attr_prefix}_name": f"Test Record {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_description": "This is a test record created by the async SDK functionality test", + f"{attr_prefix}_count": 42, + f"{attr_prefix}_amount": 123.45, + f"{attr_prefix}_is_active": True, + f"{attr_prefix}_created_date": datetime.now().isoformat(), + } + + try: + print("Creating test record...") + created_id: Optional[str] = None + for attempt in range(1, retries + 1): + try: + created_id = await client.records.create(table_schema_name, test_data) + if attempt > 1: + print(f" [OK] Record creation succeeded after {attempt} attempts.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print( + f" Table not ready for create (attempt {attempt}/{retries}). Retrying in {delay_seconds}s..." + ) + await asyncio.sleep(delay_seconds) + continue + raise + + if created_id: + print(f"[OK] Record created successfully!") + print(f" Record ID: {created_id}") + return created_id + else: + raise ValueError("Unexpected response from records.create operation") + + except Exception as e: + print(f"[ERR] Failed to create record: {e}") + sys.exit(1) + + +async def test_read_record( + client: AsyncDataverseClient, + table_info: Dict[str, Any], + record_id: str, +) -> Dict[str, Any]: + """Test record reading.""" + print("\n-> Record Reading Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + try: + print(f"Reading record: {record_id}") + record = None + for attempt in range(1, retries + 1): + try: + record = await client.records.retrieve(table_schema_name, record_id) + if attempt > 1: + print(f" [OK] Record read succeeded after {attempt} attempts.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print(f" Record not queryable yet (attempt {attempt}/{retries}). Retrying in {delay_seconds}s...") + await asyncio.sleep(delay_seconds) + continue + raise + + if record is None: + raise RuntimeError("Record did not become available in time.") + + print("[OK] Record retrieved successfully!") + for field_name in [ + f"{attr_prefix}_name", + f"{attr_prefix}_description", + f"{attr_prefix}_count", + f"{attr_prefix}_amount", + f"{attr_prefix}_is_active", + ]: + if field_name in record: + print(f" {field_name}: {record[field_name]}") + + # include_annotations + annotation = "OData.Community.Display.V1.FormattedValue" + annotated = await client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_is_active", f"{attr_prefix}_count"], + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + if annotated is not None and ann_key in annotated: + print(f"[OK] include_annotations verified: {ann_key} = '{annotated[ann_key]}'") + else: + print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + + # expand + try: + expanded = await client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_name"], + expand=["owninguser"], + ) + owner = (expanded.get("owninguser") or {}) if expanded else {} + owner_name = owner.get("fullname") or owner.get("domainname") or "(unknown)" + print(f"[OK] records.retrieve with expand=['owninguser']: owner='{owner_name}'") + except Exception as e: + print(f"[WARN] records.retrieve expand skipped: {e}") + + return record + + except Exception as e: + print(f"[ERR] Failed to read record: {e}") + sys.exit(1) + + +async def test_query_records(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> None: + """Test querying multiple records.""" + print("\n-> Record Query Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + retries = 5 + delay_seconds = 3 + + select_cols = [f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"] + active_filter = f"{attr_prefix}_is_active eq true" + + try: + # records.list() — eager + print("Querying records with await client.records.list()...") + for attempt in range(1, retries + 1): + try: + result = await client.records.list( + table_schema_name, + select=select_cols, + filter=active_filter, + top=5, + ) + record_count = 0 + for record in result: + record_count += 1 + name = record.get(f"{attr_prefix}_name", "N/A") + count = record.get(f"{attr_prefix}_count", "N/A") + amount = record.get(f"{attr_prefix}_amount", "N/A") + print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") + print(f"[OK] records.list() completed! Found {record_count} active records.") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404 and attempt < retries: + print(f" Query retry {attempt}/{retries}. Waiting {delay_seconds}s...") + await asyncio.sleep(delay_seconds) + continue + raise + + # records.list_pages() — lazy + print("\nQuerying records with async for page in client.records.list_pages() (paged)...") + page_num = 0 + total_records = 0 + async for page in client.records.list_pages( + table_schema_name, + select=select_cols, + filter=active_filter, + ): + page_num += 1 + total_records += len(page) + names = [r.get(f"{attr_prefix}_name", "N/A") for r in page] + print(f" Page {page_num}: {len(page)} record(s) — {names}") + print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + + # records.list() with extended params + print("\nQuerying records.list() with orderby / page_size / count / include_annotations...") + annotation = "OData.Community.Display.V1.FormattedValue" + annotated_result = await client.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in annotated_result) + if ann_present: + print(f"[OK] include_annotations verified: '{ann_key}' present in list() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found") + print(f"[OK] records.list() with extended params completed! {len(annotated_result)} record(s).") + + # AsyncQueryBuilder + from PowerPlatform.Dataverse.models.filters import col + + print("\nQuerying with AsyncQueryBuilder (.where(col(...)) + .page_size().execute_pages())...") + qb_pages = 0 + qb_total = 0 + async for page in ( + client.query.builder(table_schema_name) + .select(f"{attr_prefix}_name", f"{attr_prefix}_count") + .where(col(f"{attr_prefix}_is_active") == True) + .page_size(10) + .execute_pages() + ): + qb_pages += 1 + qb_total += len(page) + print(f"[OK] AsyncQueryBuilder execute_pages(): {qb_total} records across {qb_pages} page(s).") + + # FetchXML + print("\nQuerying with client.query.fetchxml().execute() ...") + fx_xml = f""" + + + + + + + + + + """ + try: + fx_result = await client.query.fetchxml(fx_xml).execute() + print(f"[OK] FetchXML execute(): {len(fx_result)} active records.") + except Exception as e: + print(f"[WARN] FetchXML query encountered an issue: {e}") + + except Exception as e: + print(f"[WARN] Query test encountered an issue: {e}") + print(" This might be expected if the table is very new.") + + +async def test_batch_all_operations(client: AsyncDataverseClient, table_info: Dict[str, Any]) -> None: + """Test batch operations using the async batch client.""" + print("\n-> Batch Operations Test") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + logical_name = table_info.get("table_logical_name", table_schema_name.lower()) + attr_prefix = table_schema_name.split("_", 1)[0] if "_" in table_schema_name else table_schema_name + all_ids: list = [] + + try: + # [1] CREATE — single + CreateMultiple + print("\n[1/7] Create — single + CreateMultiple") + batch = client.batch.new() + batch.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-A {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 1, + f"{attr_prefix}_is_active": True, + }, + ) + batch.records.create( + table_schema_name, + [ + { + f"{attr_prefix}_name": f"Batch-B {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 2, + f"{attr_prefix}_is_active": True, + }, + { + f"{attr_prefix}_name": f"Batch-C {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 3, + f"{attr_prefix}_is_active": True, + }, + ], + ) + result = await batch.execute() + all_ids = list(result.entity_ids) + if result.has_errors: + for item in result.failed: + print(f"[WARN] {item.status_code}: {item.error_message}") + else: + print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created") + + # [2] READ — retrieve + list + query.sql + if all_ids: + annotation = "OData.Community.Display.V1.FormattedValue" + print(f"\n[2/7] Read — records.retrieve + records.list + query.sql") + batch = client.batch.new() + batch.records.retrieve( + table_schema_name, + all_ids[0], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + include_annotations=annotation, + ) + batch.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=f"{attr_prefix}_is_active eq true", + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + include_annotations=annotation, + ) + batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") + result = await batch.execute() + print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") + + # [3] UPDATE — single + multiple + if len(all_ids) >= 2: + print(f"\n[3/7] Update — single PATCH + UpdateMultiple") + batch = client.batch.new() + batch.records.update(table_schema_name, all_ids[0], {f"{attr_prefix}_count": 10}) + batch.records.update(table_schema_name, all_ids[1:], {f"{attr_prefix}_count": 20}) + result = await batch.execute() + print(f"[OK] {len(result.succeeded)} updates succeeded") + + # [4] CHANGESET (happy path) — create + update via content-ID + delete + if all_ids: + print("\n[4/7] Changeset (happy path) — create + update(ref) + delete") + batch = client.batch.new() + async with batch.changeset() as cs: + ref = cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Batch-D {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 4, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, ref, {f"{attr_prefix}_is_active": True}) + cs.records.delete(table_schema_name, all_ids[-1]) + result = await batch.execute() + if result.has_errors: + for item in result.failed: + print(f"[WARN] {item.status_code}: {item.error_message}") + else: + new_id = next(iter(result.entity_ids), None) + if new_id: + all_ids[-1] = new_id + print(f"[OK] {len(result.succeeded)} ops committed atomically") + + # [5] CHANGESET (rollback) + print("\n[5/7] Changeset (rollback) — failing update rolls back create") + batch = client.batch.new() + async with batch.changeset() as cs: + cs.records.create( + table_schema_name, + { + f"{attr_prefix}_name": f"Rollback-test {datetime.now().strftime('%H:%M:%S')}", + f"{attr_prefix}_count": 0, + f"{attr_prefix}_is_active": False, + }, + ) + cs.records.update(table_schema_name, "00000000-0000-0000-0000-000000000001", {f"{attr_prefix}_count": 999}) + result = await batch.execute(continue_on_error=True) + if result.has_errors: + print("[OK] Changeset rollback verified: changeset failed, no records created") + else: + print("[WARN] Expected rollback but changeset succeeded (unexpected)") + all_ids.extend(result.entity_ids) + + # [6] ADD/REMOVE COLUMNS + col_a = f"{attr_prefix}_batch_extra_a" + col_b = f"{attr_prefix}_batch_extra_b" + print(f"\n[6/7] Batch tables.add_columns + tables.remove_columns") + batch = client.batch.new() + batch.tables.add_columns(table_schema_name, {col_a: "string"}) + batch.tables.add_columns(table_schema_name, {col_b: "int"}) + result = await batch.execute() + if not result.has_errors: + print(f"[OK] {len(result.succeeded)} column(s) added: {col_a}, {col_b}") + batch_rm = client.batch.new() + batch_rm.tables.remove_columns(table_schema_name, [col_a, col_b]) + rm_result = await batch_rm.execute(continue_on_error=True) + print(f"[OK] Removed {len(rm_result.succeeded)} batch-added column(s)") + else: + for item in result.failed: + print(f"[WARN] add_columns error {item.status_code}: {item.error_message}") + + # [7] DELETE + if all_ids: + print(f"\n[7/7] Delete — {len(all_ids)} records (use_bulk_delete=False)") + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + result = await batch.execute(continue_on_error=True) + print(f"[OK] Deleted {len(result.succeeded)}, failed {len(result.failed)}") + + print("\n[OK] Batch all-operations test completed!") + + except Exception as e: + print(f"[WARN] Batch test encountered an issue: {e}") + if all_ids: + try: + batch = client.batch.new() + batch.records.delete(table_schema_name, all_ids, use_bulk_delete=False) + await batch.execute(continue_on_error=True) + except Exception: + pass + + +async def test_relationships(client: AsyncDataverseClient) -> None: + """Test relationship lifecycle: create tables, 1:N, N:N, query, delete.""" + print("\n-> Relationship Tests") + print("=" * 50) + + rel_parent_schema = "test_RelParent" + rel_child_schema = "test_RelChild" + rel_m2m_schema = "test_RelProject" + + rel_id_1n = None + rel_id_lookup = None + rel_id_nn = None + created_tables = [] + + try: + # Cleanup leftovers + print("Checking for leftover relationship test resources...") + found_leftovers = False + for rel_name in ["test_RelParent_RelChild", "contact_test_relchild_test_ManagerId", "test_relchild_relproject"]: + try: + rel = await client.tables.get_relationship(rel_name) + if rel: + found_leftovers = True + break + except Exception: + pass + + if not found_leftovers: + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if await client.tables.get(tbl): + found_leftovers = True + break + except Exception: + pass + + if found_leftovers: + cleanup_ok = input("Found leftover test resources. Clean up? (y/N): ").strip().lower() in ["y", "yes"] + if cleanup_ok: + for rel_name in [ + "test_RelParent_RelChild", + "contact_test_relchild_test_ManagerId", + "test_relchild_relproject", + ]: + try: + rel = await client.tables.get_relationship(rel_name) + if rel: + await client.tables.delete_relationship(rel.relationship_id) + print(f" (Cleaned up relationship: {rel_name})") + except Exception: + pass + for tbl in [rel_child_schema, rel_parent_schema, rel_m2m_schema]: + try: + if await client.tables.get(tbl): + await client.tables.delete(tbl) + print(f" (Cleaned up table: {tbl})") + except Exception: + pass + + # Create tables + print("\nCreating relationship test tables...") + + async def _get_or_create(schema, columns, label): + info = await client.tables.get(schema) + if info: + print(f"[OK] Table already exists: {schema} (skipped)") + return info + try: + result = await backoff(lambda: client.tables.create(schema, columns)) + print(f"[OK] Created {label}: {schema}") + return result + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {schema} (skipped)") + return await client.tables.get(schema) + raise + + parent_info = await _get_or_create(rel_parent_schema, {"test_Code": "string"}, "parent table") + created_tables.append(rel_parent_schema) + + child_info = await _get_or_create(rel_child_schema, {"test_Number": "string"}, "child table") + created_tables.append(rel_child_schema) + + proj_info = await _get_or_create(rel_m2m_schema, {"test_ProjectCode": "string"}, "M:N table") + created_tables.append(rel_m2m_schema) + + await wait_for_table_metadata(client, rel_parent_schema) + await wait_for_table_metadata(client, rel_child_schema) + await wait_for_table_metadata(client, rel_m2m_schema) + + # 1:N relationship + print("\n Test 1: Create 1:N relationship") + lookup = LookupAttributeMetadata( + schema_name="test_ParentId", + display_name=Label(localized_labels=[LocalizedLabel(label="Parent", language_code=1033)]), + required_level="None", + ) + relationship = OneToManyRelationshipMetadata( + schema_name="test_RelParent_RelChild", + referenced_entity=parent_info["table_logical_name"], + referencing_entity=child_info["table_logical_name"], + referenced_attribute=f"{parent_info['table_logical_name']}id", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + assign=CASCADE_BEHAVIOR_NO_CASCADE, + merge=CASCADE_BEHAVIOR_NO_CASCADE, + ), + ) + existing_1n = await client.tables.get_relationship("test_RelParent_RelChild") + if existing_1n: + rel_id_1n = existing_1n.relationship_id + print(f" [OK] Relationship already exists (skipped)") + else: + result_1n = await backoff( + lambda: client.tables.create_one_to_many_relationship(lookup=lookup, relationship=relationship) + ) + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N: {result_1n.relationship_schema_name}") + + # Lookup field + print("\n Test 2: Create lookup field (convenience API)") + existing_lookup = await client.tables.get_relationship("contact_test_relchild_test_ManagerId") + if existing_lookup: + rel_id_lookup = existing_lookup.relationship_id + print(f" [OK] Lookup already exists (skipped)") + else: + result_lookup = await backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + ) + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + + # N:N relationship + print("\n Test 3: Create N:N relationship") + m2m = ManyToManyRelationshipMetadata( + schema_name="test_relchild_relproject", + entity1_logical_name=child_info["table_logical_name"], + entity2_logical_name=proj_info["table_logical_name"], + ) + existing_nn = await client.tables.get_relationship("test_relchild_relproject") + if existing_nn: + rel_id_nn = existing_nn.relationship_id + print(f" [OK] Relationship already exists (skipped)") + else: + result_nn = await backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + assert result_nn.relationship_schema_name == "test_relchild_relproject" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N: {result_nn.relationship_schema_name}") + + # Get relationship metadata + print("\n Test 4: Query relationship metadata") + fetched_1n = await client.tables.get_relationship("test_RelParent_RelChild") + assert fetched_1n is not None and fetched_1n.relationship_type == "one_to_many" + print(f" [OK] Retrieved 1:N: {fetched_1n.relationship_schema_name}") + + fetched_nn = await client.tables.get_relationship("test_relchild_relproject") + assert fetched_nn is not None and fetched_nn.relationship_type == "many_to_many" + print(f" [OK] Retrieved N:N: {fetched_nn.relationship_schema_name}") + + missing = await client.tables.get_relationship("nonexistent_relationship_xyz") + assert missing is None + print(" [OK] Non-existent relationship returns None") + + # Delete relationships + print("\n Test 5: Delete relationships") + await backoff(lambda: client.tables.delete_relationship(rel_id_1n)) + rel_id_1n = None + print(" [OK] Deleted 1:N relationship") + + await backoff(lambda: client.tables.delete_relationship(rel_id_lookup)) + rel_id_lookup = None + print(" [OK] Deleted lookup relationship") + + await backoff(lambda: client.tables.delete_relationship(rel_id_nn)) + rel_id_nn = None + print(" [OK] Deleted N:N relationship") + + verify = await client.tables.get_relationship("test_RelParent_RelChild") + assert verify is None + print(" [OK] Verified 1:N deletion") + + print("\n[OK] All relationship tests passed!") + + finally: + for rid in [rel_id_1n, rel_id_lookup, rel_id_nn]: + if rid: + try: + await client.tables.delete_relationship(rid) + except Exception: + pass + + for tbl in reversed(created_tables): + try: + await backoff(lambda name=tbl: client.tables.delete(name)) + print(f" (Cleaned up table: {tbl})") + except Exception as e: + print(f" [WARN] Could not delete {tbl}: {e}") + + +async def cleanup_test_data( + client: AsyncDataverseClient, + table_info: Dict[str, Any], + record_id: str, +) -> None: + """Clean up test data.""" + print("\n-> Cleanup") + print("=" * 50) + + table_schema_name = table_info.get("table_schema_name") + retries = 5 + delay_seconds = 3 + + cleanup_choice = input("Do you want to delete the test record? (y/N): ").strip().lower() + if cleanup_choice in ["y", "yes"]: + for attempt in range(1, retries + 1): + try: + await client.records.delete(table_schema_name, record_id) + print("[OK] Test record deleted successfully") + break + except HttpError as err: + if getattr(err, "status_code", None) == 404: + print("Record already deleted; skipping.") + break + if attempt < retries: + await asyncio.sleep(delay_seconds) + continue + print(f"[WARN] Failed to delete test record: {err}") + except Exception as e: + print(f"[WARN] Failed to delete test record: {e}") + break + else: + print("Test record kept for inspection") + + table_cleanup = input("Do you want to delete the test table? (y/N): ").strip().lower() + if table_cleanup in ["y", "yes"]: + for attempt in range(1, retries + 1): + try: + await client.tables.delete(table_schema_name) + print("[OK] Test table deleted successfully") + break + except HttpError as err: + if attempt < retries: + await asyncio.sleep(delay_seconds) + continue + print(f"[WARN] Failed to delete test table: {err}") + except Exception as e: + print(f"[WARN] Failed to delete test table: {e}") + break + else: + print("Test table kept for future testing") + + +async def main(): + """Main async test function.""" + print("PowerPlatform Dataverse Client SDK - Async Functional Testing") + print("=" * 70) + print("This script tests async SDK functionality in a real Dataverse environment:") + print(" - Authentication & Connection") + print(" - Table Creation & Metadata Operations") + print(" - Record CRUD Operations") + print(" - Query Functionality (list, list_pages, builder, fetchxml)") + print(" - Relationship Operations (1:N, N:N, lookup)") + print(" - Batch Operations (create, read, update, changeset, delete)") + print(" - Interactive Cleanup") + print("=" * 70) + print("For installation validation, run examples/aio/basic/installation_example.py first") + print("=" * 70) + + try: + client, credential = await setup_authentication() + + try: + async with client: + table_info = await ensure_test_table(client) + record_id = await test_create_record(client, table_info) + await test_read_record(client, table_info, record_id) + await test_query_records(client, table_info) + await test_relationships(client) + await test_batch_all_operations(client, table_info) + + print("\nAsync Functional Test Summary") + print("=" * 50) + print("[OK] Authentication: Success") + print("[OK] Table Operations: Success") + print("[OK] Record Creation: Success") + print("[OK] Record Reading: Success") + print("[OK] Record Querying (list, list_pages, builder, fetchxml): Success") + print("[OK] Relationship Operations: Success") + print("[OK] Batch Operations: Success") + print("\nYour async PowerPlatform Dataverse Client SDK is fully functional!") + + await cleanup_test_data(client, table_info, record_id) + finally: + await credential.close() + + except KeyboardInterrupt: + print("\n\n[WARN] Test interrupted by user") + sys.exit(1) + except Exception as e: + print(f"\n[ERR] Unexpected error: {e}") + sys.exit(1) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/aio/basic/installation_example.py b/examples/aio/basic/installation_example.py new file mode 100644 index 00000000..df958c17 --- /dev/null +++ b/examples/aio/basic/installation_example.py @@ -0,0 +1,372 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +PowerPlatform Dataverse Client - Async Installation, Validation & Usage Example + +Async equivalent of examples/basic/installation_example.py. + +This script demonstrates the async client (AsyncDataverseClient) and validates +that all async imports, classes, and methods are correctly installed. + +## Installation + +```bash +pip install PowerPlatform-Dataverse-Client azure-identity +``` + +## What This Script Does + +- Validates async package imports +- Checks version and package metadata +- Shows async usage patterns +- Offers optional interactive testing with a real Dataverse environment + +Prerequisites for Interactive Testing: +- Access to a Microsoft Dataverse environment +- Azure Identity credentials configured +- Interactive browser access for authentication +""" + +import asyncio +import sys +import subprocess +from datetime import datetime +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations + + +def validate_imports(): + """Validate that all key async imports work correctly.""" + print("Validating Async Package Imports...") + print("-" * 50) + + try: + from PowerPlatform.Dataverse import __version__ + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + print(f" [OK] Namespace: PowerPlatform.Dataverse.aio") + print(f" [OK] Package version: {__version__}") + print(f" [OK] Async client: PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient") + + from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError + + print(f" [OK] Core errors: HttpError, MetadataError") + + from PowerPlatform.Dataverse.core.config import DataverseConfig + + print(f" [OK] Core config: DataverseConfig") + + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + print(f" [OK] Async data layer: _AsyncODataClient") + + from PowerPlatform.Dataverse.aio.models.async_fetchxml_query import AsyncFetchXmlQuery + from PowerPlatform.Dataverse.aio.models.async_query_builder import AsyncQueryBuilder + + print(f" [OK] Async models: AsyncFetchXmlQuery, AsyncQueryBuilder") + + from _auth import AsyncInteractiveBrowserCredential + + print(f" [OK] Azure Identity: AsyncInteractiveBrowserCredential (interactive browser)") + + return True, __version__, AsyncDataverseClient + + except ImportError as e: + print(f" [ERR] Import failed: {e}") + print("\nTroubleshooting:") + print(" pip install PowerPlatform-Dataverse-Client azure-identity") + print(" Or for development: pip install -e .") + return False, None, None + + +def validate_client_methods(AsyncDataverseClient): + """Validate that AsyncDataverseClient has expected methods.""" + print("\nValidating Async Client Methods...") + print("-" * 50) + + expected_namespaces = { + "records": ["create", "retrieve", "update", "delete", "list", "list_pages", "upsert"], + "query": ["sql", "builder", "fetchxml", "sql_columns", "odata_expands"], + "tables": [ + "create", + "get", + "list", + "delete", + "add_columns", + "remove_columns", + "create_one_to_many_relationship", + "create_many_to_many_relationship", + "delete_relationship", + "get_relationship", + "create_lookup_field", + ], + "files": ["upload"], + } + + ns_classes = { + "records": AsyncRecordOperations, + "query": AsyncQueryOperations, + "tables": AsyncTableOperations, + "files": AsyncFileOperations, + } + + missing_methods = [] + for ns, methods in expected_namespaces.items(): + ns_cls = ns_classes.get(ns) + for method in methods: + attr_path = f"{ns}.{method}" + if ns_cls is not None and hasattr(ns_cls, method): + print(f" [OK] Method exists: {attr_path}") + else: + print(f" [ERR] Method missing: {attr_path}") + missing_methods.append(attr_path) + + return len(missing_methods) == 0 + + +def validate_package_metadata(): + """Validate package metadata from pip.""" + print("\nValidating Package Metadata...") + print("-" * 50) + + try: + result = subprocess.run( + [sys.executable, "-m", "pip", "show", "PowerPlatform-Dataverse-Client"], + capture_output=True, + text=True, + ) + if result.returncode == 0: + for line in result.stdout.split("\n"): + if any(line.startswith(p) for p in ["Name:", "Version:", "Summary:", "Location:"]): + print(f" [OK] {line}") + return True + else: + print(" [ERR] Package not found in pip list") + return False + except Exception as e: + print(f" [ERR] Metadata validation failed: {e}") + return False + + +def show_usage_examples(): + """Display async usage examples.""" + print("\nAsync Usage Examples") + print("=" * 50) + + print(""" +Basic Setup: +```python +import asyncio +import sys +from pathlib import Path +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from _auth import AsyncInteractiveBrowserCredential +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + +async def main(): + credential = AsyncInteractiveBrowserCredential() + try: + async with AsyncDataverseClient("https://yourorg.crm.dynamics.com", credential) as client: + ... # all operations here + finally: + await credential.close() + +asyncio.run(main()) +``` + +CRUD Operations: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + # Create a record + account_id = await client.records.create("account", {"name": "Contoso Ltd"}) + + # Read a single record by ID + account = await client.records.retrieve("account", account_id) + print(f"Account name: {account['name']}") + + # Update a record + await client.records.update("account", account_id, {"telephone1": "555-0200"}) + + # Delete a record + await client.records.delete("account", account_id) +``` + +Querying Data: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + from PowerPlatform.Dataverse.models.filters import col + + # Fluent query builder + result = await ( + client.query.builder("account") + .select("name", "telephone1") + .where(col("statecode") == 0) + .top(10) + .execute() + ) + for record in result: + print(record["name"]) + + # Lazy paged iteration + async for page in ( + client.query.builder("account") + .select("name") + .page_size(50) + .execute_pages() + ): + for record in page: + print(record["name"]) + + # SQL query + rows = await client.query.sql("SELECT TOP 5 name FROM account") + for row in rows: + print(row["name"]) + + # FetchXML + xml = '' + rows = await client.query.fetchxml(xml).execute() + for row in rows: + print(row["name"]) +``` + +Batch Operations: +```python +async def main(): + async with AsyncDataverseClient(url, credential) as client: + batch = client.batch.new() + batch.records.create("account", {"name": "Alpha"}) + batch.records.create("account", {"name": "Beta"}) + result = await batch.execute() + print(f"Created {len(list(result.entity_ids))} records") + + # Atomic changeset + batch = client.batch.new() + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + result = await batch.execute() +``` +""") + + +async def interactive_test(): + """Offer optional interactive testing with real Dataverse environment.""" + print("\nInteractive Testing") + print("=" * 50) + + choice = input("Would you like to test with a real Dataverse environment? (y/N): ").strip().lower() + if choice not in ["y", "yes"]: + print(" Skipping interactive test") + return + + if not sys.stdin.isatty(): + print(" [ERR] Interactive input required for testing") + return + + org_url = input("Enter your Dataverse org URL (e.g., https://yourorg.crm.dynamics.com): ").strip() + if not org_url: + print(" [WARN] No URL provided, skipping test") + return + + try: + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + from _auth import AsyncInteractiveBrowserCredential + + print(" Setting up authentication...") + credential = AsyncInteractiveBrowserCredential() + + print(" Creating async client...") + try: + async with AsyncDataverseClient(org_url.rstrip("/"), credential) as client: + print(" Testing connection...") + tables = await client.tables.list() + print(f" [OK] Connection successful!") + print(f" Found {len(tables)} tables in environment") + + custom_tables = await client.tables.list( + filter="IsCustomEntity eq true", + select=["LogicalName", "SchemaName"], + ) + print(f" Found {len(custom_tables)} custom tables (filter + select)") + finally: + await credential.close() + + print("\n Your async SDK is ready for use!") + + except Exception as e: + print(f" [ERR] Interactive test failed: {e}") + print(" This might be due to authentication, network, or permissions") + print(" The SDK imports are still valid for offline development") + + +async def main(): + """Run async installation validation and demonstration.""" + print("PowerPlatform Dataverse Client SDK - Async Installation & Validation") + print("=" * 70) + print(f"Validation Time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}") + print("=" * 70) + + imports_success, version, AsyncDataverseClient = validate_imports() + if not imports_success: + print("\n[ERR] Import validation failed. Please check installation.") + sys.exit(1) + + methods_success = True + if AsyncDataverseClient: + methods_success = validate_client_methods(AsyncDataverseClient) + if not methods_success: + print("\n[WARN] Some client methods are missing, but basic functionality should work.") + + metadata_success = validate_package_metadata() + + show_usage_examples() + + await interactive_test() + + print("\n" + "=" * 70) + print("VALIDATION SUMMARY") + print("=" * 70) + + results = [ + ("Async Package Imports", imports_success), + ("Async Client Methods", methods_success), + ("Package Metadata", metadata_success), + ] + + all_passed = True + for test_name, success in results: + status = "[OK] PASS" if success else "[ERR] FAIL" + print(f"{test_name:<25} {status}") + if not success: + all_passed = False + + print("=" * 70) + if all_passed: + print("SUCCESS: Async PowerPlatform-Dataverse-Client is properly installed!") + if version: + print(f"Package Version: {version}") + print("\nNext Steps:") + print(" - Run examples/aio/basic/functional_testing.py for a live test") + print(" - Run examples/aio/advanced/walkthrough.py for a full feature tour") + else: + print("[ERR] Some validation checks failed!") + print(" pip uninstall PowerPlatform-Dataverse-Client") + print(" pip install PowerPlatform-Dataverse-Client") + sys.exit(1) + + +if __name__ == "__main__": + print("PowerPlatform-Dataverse-Client SDK - Async Installation Example") + print("=" * 60) + asyncio.run(main()) diff --git a/examples/basic/functional_testing.py b/examples/basic/functional_testing.py index 1ea0d5f0..ddebd362 100644 --- a/examples/basic/functional_testing.py +++ b/examples/basic/functional_testing.py @@ -236,7 +236,7 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record record = None for attempt in range(1, retries + 1): try: - record = client.records.get(table_schema_name, record_id) + record = client.records.retrieve(table_schema_name, record_id) if attempt > 1: print(f" [OK] Record read succeeded after {attempt} attempts.") break @@ -250,24 +250,48 @@ def test_read_record(client: DataverseClient, table_info: Dict[str, Any], record if record is None: raise RuntimeError("Record did not become available in time.") - if record: - print("[OK] Record retrieved successfully!") - print(" Retrieved data:") - - # Display key fields - for field_name in [ - f"{attr_prefix}_name", - f"{attr_prefix}_description", - f"{attr_prefix}_count", - f"{attr_prefix}_amount", - f"{attr_prefix}_is_active", - ]: - if field_name in record: - print(f" {field_name}: {record[field_name]}") - - return record + print("[OK] Record retrieved successfully!") + print(" Retrieved data:") + for field_name in [ + f"{attr_prefix}_name", + f"{attr_prefix}_description", + f"{attr_prefix}_count", + f"{attr_prefix}_amount", + f"{attr_prefix}_is_active", + ]: + if field_name in record: + print(f" {field_name}: {record[field_name]}") + + # -- include_annotations: verify FormattedValue annotations are returned -- + annotation = "OData.Community.Display.V1.FormattedValue" + annotated = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_is_active", f"{attr_prefix}_count"], + include_annotations=annotation, + ) + ann_key = f"{attr_prefix}_is_active@{annotation}" + if annotated is not None and ann_key in annotated: + print(f"[OK] include_annotations verified: {ann_key} = '{annotated[ann_key]}'") else: - raise ValueError("Record not found") + print(f"[WARN] include_annotations: expected key '{ann_key}' not present in response") + + # -- expand: verify navigation property expansion on a single-record GET -- + # owninguser is a system navigation property present on all user-owned tables. + try: + expanded = client.records.retrieve( + table_schema_name, + record_id, + select=[f"{attr_prefix}_name"], + expand=["owninguser"], + ) + owner = (expanded.get("owninguser") or {}) if expanded else {} + owner_name = owner.get("fullname") or owner.get("domainname") or "(unknown)" + print(f"[OK] records.retrieve with expand=['owninguser']: owner='{owner_name}'") + except Exception as e: # noqa: BLE001 + print(f"[WARN] records.retrieve expand skipped: {e}") + + return record except HttpError as e: print(f"[ERR] HTTP error during record reading: {e}") @@ -287,28 +311,30 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N retries = 5 delay_seconds = 3 + select_cols = [f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"] + active_filter = f"{attr_prefix}_is_active eq true" + try: - print("Querying records from test table...") + # -- records.list() — eager, all pages collected into one QueryResult ---------- + print("Querying records with records.list()...") for attempt in range(1, retries + 1): try: - records_iterator = client.records.get( + result = client.records.list( table_schema_name, - select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_amount"], - filter=f"{attr_prefix}_is_active eq true", + select=select_cols, + filter=active_filter, top=5, - orderby=[f"{attr_prefix}_name asc"], ) record_count = 0 - for batch in records_iterator: - for record in batch: - record_count += 1 - name = record.get(f"{attr_prefix}_name", "N/A") - count = record.get(f"{attr_prefix}_count", "N/A") - amount = record.get(f"{attr_prefix}_amount", "N/A") - print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") - - print(f"[OK] Query completed! Found {record_count} active records.") + for record in result: + record_count += 1 + name = record.get(f"{attr_prefix}_name", "N/A") + count = record.get(f"{attr_prefix}_count", "N/A") + amount = record.get(f"{attr_prefix}_amount", "N/A") + print(f" Record {record_count}: {name} (Count: {count}, Amount: {amount})") + + print(f"[OK] records.list() completed! Found {record_count} active records.") break except HttpError as err: if getattr(err, "status_code", None) == 404 and attempt < retries: @@ -317,6 +343,66 @@ def test_query_records(client: DataverseClient, table_info: Dict[str, Any]) -> N continue raise + # -- records.list_pages() — lazy, one QueryResult per HTTP page --------------- + print("\nQuerying records with records.list_pages() (paged)...") + page_num = 0 + total_records = 0 + for page in client.records.list_pages( + table_schema_name, + select=select_cols, + filter=active_filter, + ): + page_num += 1 + total_records += len(page) + names = [r.get(f"{attr_prefix}_name", "N/A") for r in page] + print(f" Page {page_num}: {len(page)} record(s) — {names}") + print(f"[OK] records.list_pages() completed! {total_records} records across {page_num} page(s).") + + # -- records.list() with orderby, page_size, count, include_annotations -------- + print("\nQuerying records.list() with orderby / page_size / count / include_annotations...") + annotation = "OData.Community.Display.V1.FormattedValue" + annotated_result = client.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, + ) + names_ordered = [r.get(f"{attr_prefix}_name", "N/A") for r in annotated_result] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in annotated_result) + print(f" Records (ordered): {names_ordered}") + if ann_present: + print(f"[OK] include_annotations verified: '{ann_key}' present in list() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found — may not be supported by this environment") + # Verify orderby: names should be non-decreasing + if len(names_ordered) > 1 and all(isinstance(n, str) for n in names_ordered): + assert names_ordered == sorted(names_ordered), f"orderby asc not respected: {names_ordered}" + print(f"[OK] records.list() with extended params completed! {len(annotated_result)} record(s).") + + # -- records.list_pages() with orderby, page_size, include_annotations --------- + print("\nQuerying records.list_pages() with orderby / page_size / include_annotations...") + lp_records = [] + for page in client.records.list_pages( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=active_filter, + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + include_annotations=annotation, + ): + lp_records.extend(page) + lp_names = [r.get(f"{attr_prefix}_name", "N/A") for r in lp_records] + lp_ann_present = any(ann_key in r for r in lp_records) + if lp_ann_present: + print(f"[OK] include_annotations verified in list_pages() results") + else: + print(f"[WARN] include_annotations: '{ann_key}' not found in list_pages() results") + print(f"[OK] records.list_pages() with extended params completed! {len(lp_records)} record(s).") + except Exception as e: print(f"[WARN] Query test encountered an issue: {e}") print(" This might be expected if the table is very new.") @@ -516,19 +602,39 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f"[OK] {len(result.succeeded)} ops → {len(all_ids)} records created: {all_ids}") # ------------------------------------------------------------------- - # [2/11] READ — get by ID + tables.get + tables.list + query.sql - # All 4 reads in one batch request + # [2/11] READ — records.retrieve + records.list (with extended params) + # + tables.get + tables.list + query.sql — 1 POST $batch # ------------------------------------------------------------------- if all_ids: - print("\n[2/11] Read — records.get + tables.get + tables.list + query.sql (4 ops, 1 POST $batch)") + annotation = "OData.Community.Display.V1.FormattedValue" + print( + "\n[2/11] Read — records.retrieve + records.list(orderby/page_size/count/include_annotations)" + " + tables.get + tables.list + query.sql (5 ops, 1 POST $batch)" + ) batch = client.batch.new() - batch.records.get( + # [0] Single-record retrieve with annotations and expand + batch.records.retrieve( table_schema_name, all_ids[0], - select=[f"{attr_prefix}_name", f"{attr_prefix}_count"], + select=[f"{attr_prefix}_name", f"{attr_prefix}_count", f"{attr_prefix}_is_active"], + expand=["owninguser"], + include_annotations=annotation, + ) + # [1] Multi-record list with orderby, page_size, count, include_annotations + batch.records.list( + table_schema_name, + select=[f"{attr_prefix}_name", f"{attr_prefix}_is_active"], + filter=f"{attr_prefix}_is_active eq true", + orderby=[f"{attr_prefix}_name asc"], + page_size=50, + count=True, + include_annotations=annotation, ) + # [2] Table metadata batch.tables.get(table_schema_name) + # [3] Table list batch.tables.list() + # [4] SQL batch.query.sql(f"SELECT TOP 3 {attr_prefix}_name FROM {logical_name}") result = batch.execute() print(f"[OK] {len(result.succeeded)} succeeded, {len(result.failed)} failed") @@ -537,16 +643,30 @@ def test_batch_all_operations(client: DataverseClient, table_info: Dict[str, Any print(f" [{i}] FAILED {resp.status_code}: {resp.error_message}") continue if i == 0 and resp.data: - print( - f" records.get → name='{resp.data.get(f'{attr_prefix}_name')}', count={resp.data.get(f'{attr_prefix}_count')}" - ) + name = resp.data.get(f"{attr_prefix}_name") + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_val = resp.data.get(ann_key, "") + owner = resp.data.get("owninguser") or {} + owner_name = owner.get("fullname") or owner.get("domainname") or "" + print(f" records.retrieve → name='{name}', {ann_key}='{ann_val}'") + print(f" records.retrieve expand=['owninguser'] → owner='{owner_name}'") elif i == 1 and resp.data: + rows = resp.data.get("value", []) + names_ordered = [r.get(f"{attr_prefix}_name") for r in rows] + ann_key = f"{attr_prefix}_is_active@{annotation}" + ann_present = any(ann_key in r for r in rows) + print(f" records.list → {len(rows)} row(s), ordered: {names_ordered}") + if ann_present: + print(f" [OK] include_annotations '{ann_key}' present in batch.records.list() results") + else: + print(f" [WARN] include_annotations '{ann_key}' not found in batch.records.list() results") + elif i == 2 and resp.data: print( f" tables.get → LogicalName='{resp.data.get('LogicalName')}', EntitySet='{resp.data.get('EntitySetName')}'" ) - elif i == 2 and resp.data: - print(f" tables.list → {len(resp.data.get('value', []))} tables returned") elif i == 3 and resp.data: + print(f" tables.list → {len(resp.data.get('value', []))} tables returned") + elif i == 4 and resp.data: print(f" query.sql → {len(resp.data.get('value', []))} rows returned") # ------------------------------------------------------------------- @@ -954,32 +1074,29 @@ def test_relationships(client: DataverseClient) -> None: # --- Create parent and child tables --- print("\nCreating relationship test tables...") - parent_info = backoff( - lambda: client.tables.create( - rel_parent_schema, - {"test_Code": "string"}, - ) - ) + def _get_or_create(schema, columns, label): + info = client.tables.get(schema) + if info: + print(f"[OK] Table already exists: {schema} (skipped)") + return info + try: + result = backoff(lambda: client.tables.create(schema, columns)) + print(f"[OK] Created {label}: {schema}") + return result + except Exception as e: + if "already exists" in str(e).lower() or "not unique" in str(e).lower(): + print(f"[OK] Table already exists: {schema} (skipped)") + return client.tables.get(schema) + raise + + parent_info = _get_or_create(rel_parent_schema, {"test_Code": "string"}, "parent table") created_tables.append(rel_parent_schema) - print(f"[OK] Created parent table: {parent_info['table_schema_name']}") - child_info = backoff( - lambda: client.tables.create( - rel_child_schema, - {"test_Number": "string"}, - ) - ) + child_info = _get_or_create(rel_child_schema, {"test_Number": "string"}, "child table") created_tables.append(rel_child_schema) - print(f"[OK] Created child table: {child_info['table_schema_name']}") - proj_info = backoff( - lambda: client.tables.create( - rel_m2m_schema, - {"test_ProjectCode": "string"}, - ) - ) + proj_info = _get_or_create(rel_m2m_schema, {"test_ProjectCode": "string"}, "M:N table") created_tables.append(rel_m2m_schema) - print(f"[OK] Created M:N table: {proj_info['table_schema_name']}") # --- Wait for table metadata to propagate --- wait_for_table_metadata(client, rel_parent_schema) @@ -1008,42 +1125,52 @@ def test_relationships(client: DataverseClient) -> None: ), ) - result_1n = backoff( - lambda: client.tables.create_one_to_many_relationship( - lookup=lookup, - relationship=relationship, + existing_1n = client.tables.get_relationship("test_RelParent_RelChild") + if existing_1n: + result_1n = existing_1n + rel_id_1n = result_1n.relationship_id + print(f" [OK] Relationship already exists: {result_1n.relationship_schema_name} (skipped)") + else: + result_1n = backoff( + lambda: client.tables.create_one_to_many_relationship( + lookup=lookup, + relationship=relationship, + ) ) - ) - - assert result_1n.relationship_schema_name == "test_RelParent_RelChild" - assert result_1n.relationship_type == "one_to_many" - assert result_1n.lookup_schema_name is not None - rel_id_1n = result_1n.relationship_id - print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") - print(f" Lookup: {result_1n.lookup_schema_name}") - print(f" ID: {rel_id_1n}") + assert result_1n.relationship_schema_name == "test_RelParent_RelChild" + assert result_1n.relationship_type == "one_to_many" + assert result_1n.lookup_schema_name is not None + rel_id_1n = result_1n.relationship_id + print(f" [OK] Created 1:N relationship: {result_1n.relationship_schema_name}") + print(f" Lookup: {result_1n.lookup_schema_name}") + print(f" ID: {rel_id_1n}") # --- Test 2: Create lookup field (convenience API) --- print("\n Test 2: Create lookup field (convenience API)") print(" " + "-" * 45) - result_lookup = backoff( - lambda: client.tables.create_lookup_field( - referencing_table=child_info["table_logical_name"], - lookup_field_name="test_ManagerId", - referenced_table="contact", - display_name="Manager", - description="The record's manager contact", - required=False, - cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + existing_lookup = client.tables.get_relationship("contact_test_relchild_test_ManagerId") + if existing_lookup: + result_lookup = existing_lookup + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Lookup already exists: {result_lookup.relationship_schema_name} (skipped)") + else: + result_lookup = backoff( + lambda: client.tables.create_lookup_field( + referencing_table=child_info["table_logical_name"], + lookup_field_name="test_ManagerId", + referenced_table="contact", + display_name="Manager", + description="The record's manager contact", + required=False, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) ) - ) - - assert result_lookup.relationship_type == "one_to_many" - assert result_lookup.lookup_schema_name is not None - rel_id_lookup = result_lookup.relationship_id - print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") - print(f" Relationship: {result_lookup.relationship_schema_name}") + assert result_lookup.relationship_type == "one_to_many" + assert result_lookup.lookup_schema_name is not None + rel_id_lookup = result_lookup.relationship_id + print(f" [OK] Created lookup: {result_lookup.lookup_schema_name}") + print(f" Relationship: {result_lookup.relationship_schema_name}") # --- Test 3: Create N:N relationship --- print("\n Test 3: Create N:N relationship") @@ -1055,13 +1182,18 @@ def test_relationships(client: DataverseClient) -> None: entity2_logical_name=proj_info["table_logical_name"], ) - result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) - - assert result_nn.relationship_schema_name == "test_relchild_relproject" - assert result_nn.relationship_type == "many_to_many" - rel_id_nn = result_nn.relationship_id - print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") - print(f" ID: {rel_id_nn}") + existing_nn = client.tables.get_relationship("test_relchild_relproject") + if existing_nn: + result_nn = existing_nn + rel_id_nn = result_nn.relationship_id + print(f" [OK] Relationship already exists: {result_nn.relationship_schema_name} (skipped)") + else: + result_nn = backoff(lambda: client.tables.create_many_to_many_relationship(relationship=m2m)) + assert result_nn.relationship_schema_name == "test_relchild_relproject" + assert result_nn.relationship_type == "many_to_many" + rel_id_nn = result_nn.relationship_id + print(f" [OK] Created N:N relationship: {result_nn.relationship_schema_name}") + print(f" ID: {rel_id_nn}") # --- Test 4: Get relationship metadata --- print("\n Test 4: Query relationship metadata") diff --git a/examples/basic/installation_example.py b/examples/basic/installation_example.py index 98189e58..61da149b 100644 --- a/examples/basic/installation_example.py +++ b/examples/basic/installation_example.py @@ -221,7 +221,7 @@ def show_usage_examples(): print(f"Created account: {account_id}") # Read a single record by ID -account = client.records.get("account", account_id) +account = client.records.retrieve("account", account_id) print(f"Account name: {account['name']}") # Update a record @@ -234,14 +234,13 @@ def show_usage_examples(): Querying Data: ```python # Query with OData filter -accounts = client.records.get("account", +accounts = client.records.list("account", filter="name eq 'Contoso Ltd'", select=["name", "telephone1"], top=10) -for batch in accounts: - for account in batch: - print(f"Account: {account['name']}") +for account in accounts: + print(f"Account: {account['name']}") # SQL queries (if enabled) results = client.query.sql("SELECT TOP 5 name FROM account") diff --git a/pyproject.toml b/pyproject.toml index 723e045d..1ddcc51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,24 +40,30 @@ dependencies = [ [project.scripts] dataverse-install-claude-skill = "PowerPlatform.Dataverse._skill_installer:main" +dataverse-migrate = "tools.migrate_v0_to_v1:main" [project.optional-dependencies] +async = [ + "aiohttp>=3.9", +] dev = [ "pytest>=7.0.0", "pytest-cov>=4.0.0", + "pytest-asyncio>=0.23.0", "black>=23.0.0", "isort>=5.12.0", "mypy>=1.0.0", "ruff>=0.1.0", ] +migration = ["libcst>=1.0.0"] [tool.setuptools] -package-dir = {"" = "src"} +package-dir = {"" = "src", "tools" = "tools"} zip-safe = false [tool.setuptools.packages.find] -where = ["src"] -include = ["PowerPlatform*"] +where = ["src", "."] +include = ["PowerPlatform*", "tools"] namespaces = false [tool.setuptools.package-data] @@ -93,9 +99,14 @@ select = [ [tool.pytest.ini_options] testpaths = ["tests/unit"] +asyncio_mode = "auto" [tool.coverage.run] source = ["src/PowerPlatform"] +omit = [ + "*/Dataverse/_skill_installer.py", + "*/Dataverse/extensions/__init__.py", +] [tool.coverage.report] fail_under = 90 diff --git a/src/PowerPlatform/Dataverse/__init__.py b/src/PowerPlatform/Dataverse/__init__.py index 95b5171c..0d4ed6e2 100644 --- a/src/PowerPlatform/Dataverse/__init__.py +++ b/src/PowerPlatform/Dataverse/__init__.py @@ -3,6 +3,10 @@ from importlib.metadata import version +from .models.filters import col, raw +from .models.protocol import DataverseModel +from .models.record import QueryResult + __version__ = version("PowerPlatform-Dataverse-Client") -__all__ = ["__version__"] +__all__ = ["__version__", "col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/aio/__init__.py b/src/PowerPlatform/Dataverse/aio/__init__.py new file mode 100644 index 00000000..ab39e8d8 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/__init__.py @@ -0,0 +1,12 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async namespace for the PowerPlatform Dataverse SDK. + +Import the async client via:: + + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +""" + +__all__ = [] diff --git a/src/PowerPlatform/Dataverse/aio/async_client.py b/src/PowerPlatform/Dataverse/aio/async_client.py new file mode 100644 index 00000000..30553883 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/async_client.py @@ -0,0 +1,252 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +from __future__ import annotations + +from contextlib import asynccontextmanager +from typing import AsyncIterator, Optional + +import aiohttp +from azure.core.credentials_async import AsyncTokenCredential + +from .core._async_auth import _AsyncAuthManager +from ..core.config import DataverseConfig, OperationContext +from .data._async_odata import _AsyncODataClient +from .operations.async_dataframe import AsyncDataFrameOperations +from .operations.async_records import AsyncRecordOperations +from .operations.async_query import AsyncQueryOperations +from .operations.async_files import AsyncFileOperations +from .operations.async_tables import AsyncTableOperations +from .operations.async_batch import AsyncBatchOperations + + +class AsyncDataverseClient: + """ + Async high-level client for Microsoft Dataverse operations. + + This client provides a simple, stable async interface for interacting with + Dataverse environments through the Web API. It handles authentication via + Azure Identity and delegates HTTP operations to an internal + :class:`~PowerPlatform.Dataverse.aio.data._async_odata._AsyncODataClient`. + + Key capabilities: + - OData CRUD operations: create, read, update, delete records + - SQL queries: execute read-only SQL via Web API ``?sql`` parameter + - Table metadata: create, inspect, and delete custom tables; create and delete columns + - File uploads: upload files to file columns with chunking support + + :param base_url: Your Dataverse environment URL, for example + ``"https://org.crm.dynamics.com"``. Trailing slash is automatically removed. + :type base_url: :class:`str` + :param credential: Azure async Identity credential for authentication. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential + :param config: Optional configuration for language, timeouts, and retries. + If not provided, defaults are loaded from :meth:`~PowerPlatform.Dataverse.core.config.DataverseConfig.from_env`. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig or None + :param context: Optional caller-defined context object appended to the + outbound ``User-Agent`` header for plugin/tool attribution. Cannot be used + together with ``config`` -- pass the context via + :class:`~PowerPlatform.Dataverse.core.config.DataverseConfig` instead. + :type context: ~PowerPlatform.Dataverse.core.config.OperationContext or None + + :raises ValueError: If ``base_url`` is missing or empty after trimming. + :raises ValueError: If both ``config`` and ``context`` are provided. + + .. note:: + The client lazily initializes its internal OData client on first use, + allowing lightweight construction without immediate network calls. + + .. note:: + All methods that communicate with the Dataverse Web API may raise + :class:`~PowerPlatform.Dataverse.core.errors.HttpError` on non-successful + HTTP responses (e.g. 401, 403, 404, 429, 500). Individual method + docstrings document only domain-specific exceptions. + + Operations are organized into namespaces: + + - ``client.records`` -- create, update, delete, and get records (single or paginated queries) + - ``client.query`` -- query and search operations + - ``client.tables`` -- table and column metadata management + - ``client.files`` -- file upload operations + - ``client.dataframe`` -- pandas DataFrame wrappers for record CRUD + - ``client.batch`` -- batch multiple operations into a single HTTP request + + The client supports Python's async context manager protocol for automatic + resource cleanup and HTTP connection pooling: + + Example: + **Recommended -- async context manager** (enables HTTP connection pooling):: + + from azure.identity.aio import InteractiveBrowserCredential + from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + credential = InteractiveBrowserCredential() + + async with AsyncDataverseClient("https://org.crm.dynamics.com", credential) as client: + record_id = await client.records.create("account", {"name": "Contoso Ltd"}) + await client.records.update("account", record_id, {"telephone1": "555-0100"}) + # Session closed, caches cleared automatically + + **Manual lifecycle**:: + + client = AsyncDataverseClient("https://org.crm.dynamics.com", credential) + try: + record_id = await client.records.create("account", {"name": "Contoso Ltd"}) + finally: + await client.aclose() + """ + + def __init__( + self, + base_url: str, + credential: AsyncTokenCredential, + config: Optional[DataverseConfig] = None, + *, + context: Optional[OperationContext] = None, + ) -> None: + if config is not None and context is not None: + raise ValueError( + "Cannot specify both 'config' and 'context'. " "Pass operation_context via DataverseConfig instead." + ) + self.auth = _AsyncAuthManager(credential) + self._base_url = (base_url or "").rstrip("/") + if not self._base_url: + raise ValueError("base_url is required.") + if config is not None: + self._config = config + elif context is not None: + self._config = DataverseConfig(operation_context=context) + else: + self._config = DataverseConfig.from_env() + self._odata: Optional[_AsyncODataClient] = None + self._session: Optional[aiohttp.ClientSession] = None + self._closed: bool = False + + # Operation namespaces + self.records = AsyncRecordOperations(self) + self.query = AsyncQueryOperations(self) + self.tables = AsyncTableOperations(self) + self.files = AsyncFileOperations(self) + self.dataframe = AsyncDataFrameOperations(self) + self.batch = AsyncBatchOperations(self) + + def _get_odata(self) -> _AsyncODataClient: + """ + Get or create the internal async OData client instance. + + This method implements lazy initialization of the low-level async OData + client, deferring construction until the first API call. When used outside + of an ``async with`` block, a :class:`aiohttp.ClientSession` is created + lazily here so that standalone usage (without a context manager) works + without requiring the caller to manage the session explicitly. + + :return: The lazily-initialized low-level async client. + :rtype: ~PowerPlatform.Dataverse.aio.data._async_odata._AsyncODataClient + """ + if self._odata is None: + if self._session is None: + self._session = aiohttp.ClientSession() + self._odata = _AsyncODataClient( + self.auth, + self._base_url, + self._config, + session=self._session, + ) + return self._odata + + @asynccontextmanager + async def _scoped_odata(self) -> AsyncIterator[_AsyncODataClient]: + """Async context manager yielding the low-level client with a correlation scope.""" + self._check_closed() + od = self._get_odata() + # _call_scope() is a sync context manager (just sets a context var — no I/O). + with od._call_scope(): + yield od + + # ---------------- Context manager / lifecycle ---------------- + + async def __aenter__(self) -> "AsyncDataverseClient": + """Enter the async context manager. + + Creates an :class:`aiohttp.ClientSession` for HTTP connection pooling. + All operations within the ``async with`` block reuse this session for + better performance (TCP and TLS reuse). + + :return: The client instance. + :rtype: AsyncDataverseClient + + :raises RuntimeError: If the client has been closed. + """ + self._check_closed() + if self._session is None: + self._session = aiohttp.ClientSession() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + """Exit the async context manager with cleanup. + + Calls :meth:`aclose` to release resources. Exceptions are not + suppressed. + """ + await self.aclose() + + async def aclose(self) -> None: + """Close the async client and release resources. + + Closes the HTTP session (if any), clears internal caches, and + marks the client as closed. Safe to call multiple times. After + closing, any operation will raise :class:`RuntimeError`. + + Called automatically when using the client as an async context manager. + + Example:: + + client = AsyncDataverseClient(base_url, credential) + try: + await client.records.create("account", {"name": "Contoso"}) + finally: + await client.aclose() + """ + if self._closed: + return + if self._odata is not None: + await self._odata.close() + self._odata = None + if self._session is not None: + await self._session.close() + self._session = None + self._closed = True + + def _check_closed(self) -> None: + """Raise :class:`RuntimeError` if the client has been closed.""" + if self._closed: + raise RuntimeError("AsyncDataverseClient is closed") + + # ---------------- Cache utilities ---------------- + + async def flush_cache(self, kind: str) -> int: + """ + Flush cached client metadata or state. + + :param kind: Cache kind to flush. Currently supported values: + + - ``"picklist"``: Clears picklist label cache used for label-to-integer conversion + + Future kinds (e.g. ``"entityset"``, ``"primaryid"``) may be added without + breaking this signature. + :type kind: :class:`str` + + :return: Number of cache entries removed. + :rtype: :class:`int` + + Example: + Clear the picklist cache:: + + removed = await client.flush_cache("picklist") + print(f"Cleared {removed} cached picklist entries") + """ + async with self._scoped_odata() as od: + return od._flush_cache(kind) + + +__all__ = ["AsyncDataverseClient"] diff --git a/src/PowerPlatform/Dataverse/aio/core/__init__.py b/src/PowerPlatform/Dataverse/aio/core/__init__.py new file mode 100644 index 00000000..2f0cb105 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async core infrastructure components for the Dataverse SDK. + +This module contains the foundational async components including authentication, +HTTP client, and error handling. +""" diff --git a/src/PowerPlatform/Dataverse/aio/core/_async_auth.py b/src/PowerPlatform/Dataverse/aio/core/_async_auth.py new file mode 100644 index 00000000..ca25d5b1 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/_async_auth.py @@ -0,0 +1,45 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async authentication helpers for Dataverse. + +This module provides :class:`~PowerPlatform.Dataverse.aio.core._async_auth._AsyncAuthManager`, +a thin wrapper over any Azure Identity ``AsyncTokenCredential`` for acquiring OAuth2 access +tokens asynchronously, and reuses :class:`~PowerPlatform.Dataverse.core._auth._TokenPair` for +storing the acquired token alongside its scope. +""" + +from __future__ import annotations + +from azure.core.credentials_async import AsyncTokenCredential + +from ...core._auth import _TokenPair + + +class _AsyncAuthManager: + """ + Azure Identity-based async authentication manager for Dataverse. + + :param credential: Azure Identity async credential implementation. + :type credential: ~azure.core.credentials_async.AsyncTokenCredential + :raises TypeError: If ``credential`` does not implement :class:`~azure.core.credentials_async.AsyncTokenCredential`. + """ + + def __init__(self, credential: AsyncTokenCredential) -> None: + if not isinstance(credential, AsyncTokenCredential): + raise TypeError("credential must implement azure.core.credentials_async.AsyncTokenCredential.") + self.credential: AsyncTokenCredential = credential + + async def _acquire_token(self, scope: str) -> _TokenPair: + """ + Acquire an access token asynchronously for the specified OAuth2 scope. + + :param scope: OAuth2 scope string, typically ``"https://.crm.dynamics.com/.default"``. + :type scope: :class:`str` + :return: Token pair containing the scope and access token. + :rtype: ~PowerPlatform.Dataverse.core._auth._TokenPair + :raises ~azure.core.exceptions.ClientAuthenticationError: If token acquisition fails. + """ + token = await self.credential.get_token(scope) + return _TokenPair(resource=scope, access_token=token.token) diff --git a/src/PowerPlatform/Dataverse/aio/core/_async_http.py b/src/PowerPlatform/Dataverse/aio/core/_async_http.py new file mode 100644 index 00000000..402be8d4 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/core/_async_http.py @@ -0,0 +1,185 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async HTTP client with automatic retry logic and timeout handling. + +This module provides :class:`~PowerPlatform.Dataverse.aio.core._async_http._AsyncHttpClient`, +a wrapper around the aiohttp library that adds configurable retry behavior for transient +network errors and intelligent timeout management based on HTTP method types. +""" + +from __future__ import annotations + +import asyncio +import json as _json +import time +from typing import TYPE_CHECKING, Any, Dict, Optional + +import aiohttp + +if TYPE_CHECKING: + from ...core._http_logger import _HttpLogger + + +class _AsyncResponse: + """Materialized HTTP response returned by :class:`_AsyncHttpClient._request`. + + The body is fully buffered before this object is constructed, so all + accessors are synchronous — no ``await`` required. + + :param status: HTTP status code. + :param headers: Response headers as a plain dict. + :param body: Raw response body bytes. + """ + + __slots__ = ("status", "status_code", "headers", "_body") + + def __init__(self, status: int, headers: Dict[str, str], body: bytes) -> None: + self.status = status + self.status_code = status + self.headers = headers + self._body = body + + @property + def text(self) -> str: + """Response body decoded as UTF-8 text.""" + return self._body.decode("utf-8", errors="replace") if self._body else "" + + def json(self, content_type: Any = None) -> Any: + """Parse and return the response body as JSON.""" + return _json.loads(self._body) if self._body else {} + + +class _AsyncHttpClient: + """ + Async HTTP client with configurable retry logic and timeout handling. + + Provides automatic retry behavior for transient failures and default timeout + management for different HTTP methods. + + :param retries: Maximum number of retry attempts for transient errors. Default is 5. + :type retries: :class:`int` | None + :param backoff: Base delay in seconds between retry attempts. Default is 0.5. + :type backoff: :class:`float` | None + :param timeout: Default request timeout in seconds. If None, uses per-method defaults. + :type timeout: :class:`float` | None + :param session: ``aiohttp.ClientSession`` for HTTP connection pooling. + The session is owned by the caller (``AsyncDataverseClient``) and must remain + open for the lifetime of this client. Unlike the sync client, there is no + per-request fallback — a session must always be provided before making requests. + :type session: :class:`aiohttp.ClientSession` | None + :param logger: Optional HTTP diagnostics logger. When provided, all requests, + responses, and transport errors are logged with automatic header redaction. + :type logger: ~PowerPlatform.Dataverse.core._http_logger._HttpLogger | None + """ + + def __init__( + self, + retries: Optional[int] = None, + backoff: Optional[float] = None, + timeout: Optional[float] = None, + session: Optional[aiohttp.ClientSession] = None, + logger: Optional["_HttpLogger"] = None, + ) -> None: + self.max_attempts = retries if retries is not None else 5 + self.base_delay = backoff if backoff is not None else 0.5 + self.default_timeout: Optional[float] = timeout + self._session = session + self._logger = logger + + async def _request(self, method: str, url: str, **kwargs: Any) -> _AsyncResponse: + """ + Execute an HTTP request asynchronously with automatic retry logic and timeout management. + + Applies default timeouts based on HTTP method (120s for POST/DELETE, 10s for others) + and retries on network errors with exponential backoff. + + The response body is fully buffered and returned as a :class:`_AsyncResponse` whose + accessors (``.text``, ``.json()``) are synchronous — no ``await`` required on the caller side. + + :param method: HTTP method (GET, POST, PUT, DELETE, etc.). + :type method: :class:`str` + :param url: Target URL for the request. + :type url: :class:`str` + :param kwargs: Additional arguments passed to ``aiohttp.ClientSession.request()``, + including headers, data, etc. + :return: Materialized HTTP response with body fully buffered. + :rtype: :class:`_AsyncResponse` + :raises aiohttp.ClientError: If all retry attempts fail. + :raises RuntimeError: If no session has been set. + """ + if self._session is None: + raise RuntimeError("No aiohttp.ClientSession set. Set _session before making requests.") + + # If no timeout is provided, use the user-specified default timeout if set; + # otherwise, apply per-method defaults (120s for POST/DELETE, 10s for others). + if "timeout" not in kwargs: + if self.default_timeout is not None: + t = self.default_timeout + else: + m = (method or "").lower() + t = 120 if m in ("post", "delete") else 10 + kwargs["timeout"] = aiohttp.ClientTimeout(total=t) + + # Log outbound request once (before retry loop). + # Use explicit key presence checks so falsy values (e.g. {}) are logged correctly. + if self._logger is not None: + if "json" in kwargs: + req_body = kwargs["json"] + elif "data" in kwargs: + req_body = kwargs["data"] + else: + req_body = None + self._logger.log_request( + method, + url, + headers=kwargs.get("headers"), + body=req_body, + ) + + # Small backoff retry on network errors only + for attempt in range(self.max_attempts): + try: + t0 = time.monotonic() + async with self._session.request(method, url, **kwargs) as resp: + body = await resp.read() + materialized = _AsyncResponse(resp.status, dict(resp.headers), body) + elapsed_ms = (time.monotonic() - t0) * 1000 + + if self._logger is not None: + # Only decode text when body logging is enabled — avoids + # unnecessary overhead for large payloads when max_body_bytes == 0. + resp_body = materialized.text if self._logger.body_logging_enabled else None + self._logger.log_response( + method, + url, + status_code=materialized.status, + headers=materialized.headers, + body=resp_body, + elapsed_ms=elapsed_ms, + ) + return materialized + except (aiohttp.ClientError, asyncio.TimeoutError) as exc: + if self._logger is not None: + self._logger.log_error( + method, + url, + exc, + attempt=attempt + 1, + max_attempts=self.max_attempts, + ) + if attempt == self.max_attempts - 1: + raise + delay = self.base_delay * (2**attempt) + await asyncio.sleep(delay) + continue + + async def close(self) -> None: + """Close the HTTP client and release resources. + + If a session was provided, closes it. Safe to call multiple times. + """ + if self._session is not None: + await self._session.close() + self._session = None diff --git a/src/PowerPlatform/Dataverse/aio/data/__init__.py b/src/PowerPlatform/Dataverse/aio/data/__init__.py new file mode 100644 index 00000000..be8f223c --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/__init__.py @@ -0,0 +1,9 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async data access layer for the Dataverse SDK. + +This module contains async OData protocol handling, CRUD operations, metadata management, +SQL query functionality, and file upload capabilities. +""" diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_batch.py b/src/PowerPlatform/Dataverse/aio/data/_async_batch.py new file mode 100644 index 00000000..967929dd --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_batch.py @@ -0,0 +1,295 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async batch intent resolver and dispatcher for the Dataverse Web API.""" + +from __future__ import annotations + +import uuid +from typing import TYPE_CHECKING, Any, List, Union + +from ...core.errors import MetadataError, ValidationError +from ...core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND +from ...models.batch import BatchResult +from ...data._raw_request import _RawRequest +from ...data._batch_base import ( + _BatchBase, + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, + _ChangeSetBatchItem, + _MAX_BATCH_SIZE, +) + +if TYPE_CHECKING: + from ._async_odata import _AsyncODataClient + +__all__ = [] + + +# --------------------------------------------------------------------------- +# Batch client: resolves intents → raw requests → multipart body → HTTP → result +# --------------------------------------------------------------------------- + + +class _AsyncBatchClient(_BatchBase): + """ + Async version of the Dataverse batch client. + + Serialises a list of intent objects into an OData ``$batch`` multipart/mixed + request, dispatches it asynchronously, and parses the response. + + :param od: The active async OData client (provides helpers and HTTP transport). + """ + + # ------------------------------------------------------------------ + # Public entry point + # ------------------------------------------------------------------ + + async def execute( + self, + items: List[Any], + continue_on_error: bool = False, + ) -> BatchResult: + """ + Resolve all intent objects, build the batch body, send it, and return results. + + Metadata pre-resolution (GET calls for MetadataId) happens here, asynchronously, + before the multipart body is assembled. + """ + if not items: + return BatchResult() + + resolved = await self._resolve_all(items) + + total = sum(len(r.requests) if isinstance(r, _ChangeSetBatchItem) else 1 for r in resolved) + if total > _MAX_BATCH_SIZE: + raise ValidationError( + f"Batch contains {total} operations, which exceeds the limit of " + f"{_MAX_BATCH_SIZE}. Split into multiple batches.", + subcode="batch_size_exceeded", + details={"count": total, "max": _MAX_BATCH_SIZE}, + ) + + batch_boundary = f"batch_{uuid.uuid4()}" + body = self._build_batch_body(resolved, batch_boundary) + + headers: dict[str, str] = { + "Content-Type": f'multipart/mixed; boundary="{batch_boundary}"', + } + if continue_on_error: + headers["Prefer"] = "odata.continue-on-error" + + url = f"{self._od.api}/$batch" + r = await self._od._request( + "post", + url, + data=body.encode("utf-8"), + headers=headers, + # 400 is expected: Dataverse returns 400 for top-level batch + # errors (e.g. malformed body). We parse the response body to + # surface the service error via _parse_batch_response / + # _raise_top_level_batch_error rather than letting _request raise. + expected=(200, 202, 207, 400), + ) + + return self._parse_batch_response(r) + + # ------------------------------------------------------------------ + # Intent resolution dispatcher + # ------------------------------------------------------------------ + + async def _resolve_all(self, items: List[Any]) -> List[Union[_RawRequest, _ChangeSetBatchItem]]: + result: List[Union[_RawRequest, _ChangeSetBatchItem]] = [] + for item in items: + if isinstance(item, _ChangeSet): + if not item.operations: + # Empty changeset — nothing to send; skip silently. + continue + cs_requests: List[_RawRequest] = [] + for op in item.operations: + cs_requests.append(await self._resolve_one(op)) + result.append(_ChangeSetBatchItem(requests=cs_requests)) + else: + result.extend(await self._resolve_item(item)) + return result + + async def _resolve_item(self, item: Any) -> List[_RawRequest]: + """Resolve a single intent to one or more _RawRequest objects.""" + if isinstance(item, _RecordCreate): + return await self._resolve_record_create(item) + if isinstance(item, _RecordUpdate): + return await self._resolve_record_update(item) + if isinstance(item, _RecordDelete): + return await self._resolve_record_delete(item) + if isinstance(item, _RecordGet): + return await self._resolve_record_get(item) + if isinstance(item, _RecordList): + return await self._resolve_record_list(item) + if isinstance(item, _RecordUpsert): + return await self._resolve_record_upsert(item) + if isinstance(item, _TableCreate): + return self._resolve_table_create(item) # sync; inherited from _BatchBase + if isinstance(item, _TableDelete): + return await self._resolve_table_delete(item) + if isinstance(item, _TableGet): + return self._resolve_table_get(item) # sync; inherited from _BatchBase + if isinstance(item, _TableList): + return self._resolve_table_list(item) # sync; inherited from _BatchBase + if isinstance(item, _TableAddColumns): + return await self._resolve_table_add_columns(item) + if isinstance(item, _TableRemoveColumns): + return await self._resolve_table_remove_columns(item) + if isinstance(item, _TableCreateOneToMany): + return self._resolve_table_create_one_to_many(item) # sync; inherited from _BatchBase + if isinstance(item, _TableCreateManyToMany): + return self._resolve_table_create_many_to_many(item) # sync; inherited from _BatchBase + if isinstance(item, _TableDeleteRelationship): + return self._resolve_table_delete_relationship(item) # sync; inherited from _BatchBase + if isinstance(item, _TableGetRelationship): + return self._resolve_table_get_relationship(item) # sync; inherited from _BatchBase + if isinstance(item, _TableCreateLookupField): + return self._resolve_table_create_lookup_field(item) # sync; inherited from _BatchBase + if isinstance(item, _QuerySql): + return await self._resolve_query_sql(item) + raise ValidationError( + f"Unknown batch item type: {type(item).__name__}", + subcode="unknown_batch_item", + ) + + async def _resolve_one(self, item: Any) -> _RawRequest: + """Resolve a changeset operation to exactly one _RawRequest.""" + resolved = await self._resolve_item(item) + if len(resolved) != 1: + raise ValidationError( + "Changeset operations must each produce exactly one HTTP request.", + subcode="changeset_multi_request", + ) + return resolved[0] + + # ------------------------------------------------------------------ + # Record resolvers — delegate to _AsyncODataClient._build_* methods + # ------------------------------------------------------------------ + + async def _resolve_record_create(self, op: _RecordCreate) -> List[_RawRequest]: + entity_set = await self._od._entity_set_from_schema_name(op.table) + if isinstance(op.data, dict): + return [await self._od._build_create(entity_set, op.table, op.data, content_id=op.content_id)] + return [await self._od._build_create_multiple(entity_set, op.table, op.data)] + + async def _resolve_record_update(self, op: _RecordUpdate) -> List[_RawRequest]: + if isinstance(op.ids, str): + if not isinstance(op.changes, dict): + raise TypeError("For single id, changes must be a dict") + return [await self._od._build_update(op.table, op.ids, op.changes, content_id=op.content_id)] + entity_set = await self._od._entity_set_from_schema_name(op.table) + return [await self._od._build_update_multiple(entity_set, op.table, op.ids, op.changes)] + + async def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: + if isinstance(op.ids, str): + return [await self._od._build_delete(op.table, op.ids, content_id=op.content_id)] + ids = [rid for rid in op.ids if rid] + if not ids: + return [] + if op.use_bulk_delete: + return [await self._od._build_delete_multiple(op.table, ids)] + return [await self._od._build_delete(op.table, rid) for rid in ids] + + async def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: + return [ + await self._od._build_get( + op.table, + op.record_id, + select=op.select, + expand=op.expand, + include_annotations=op.include_annotations, + ) + ] + + async def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: + return [ + await self._od._build_list( + op.table, + select=op.select, + filter=op.filter, + orderby=op.orderby, + top=op.top, + expand=op.expand, + page_size=op.page_size, + count=op.count, + include_annotations=op.include_annotations, + ) + ] + + async def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: + entity_set = await self._od._entity_set_from_schema_name(op.table) + if len(op.items) == 1: + item = op.items[0] + return [await self._od._build_upsert(entity_set, op.table, item.alternate_key, item.record)] + alternate_keys = [i.alternate_key for i in op.items] + records = [i.record for i in op.items] + return [await self._od._build_upsert_multiple(entity_set, op.table, alternate_keys, records)] + + # ------------------------------------------------------------------ + # Table resolvers — delegate to _AsyncODataClient._build_* methods + # (pre-resolution GETs for MetadataId remain here; they are batch- + # specific lookups needed before the relevant _build_* call) + # ------------------------------------------------------------------ + + async def _require_entity_metadata(self, table: str) -> str: + """Look up MetadataId for *table*, raising MetadataError if not found.""" + ent = await self._od._get_entity_by_table_schema_name(table) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + return ent["MetadataId"] + + async def _resolve_table_delete(self, op: _TableDelete) -> List[_RawRequest]: + metadata_id = await self._require_entity_metadata(op.table) + return [self._od._build_delete_entity(metadata_id)] + + async def _resolve_table_add_columns(self, op: _TableAddColumns) -> List[_RawRequest]: + metadata_id = await self._require_entity_metadata(op.table) + return [self._od._build_create_column(metadata_id, col_name, dtype) for col_name, dtype in op.columns.items()] + + async def _resolve_table_remove_columns(self, op: _TableRemoveColumns) -> List[_RawRequest]: + columns = [op.columns] if isinstance(op.columns, str) else list(op.columns) + metadata_id = await self._require_entity_metadata(op.table) + attr_metas = [ + await self._od._get_attribute_metadata(metadata_id, col_name, extra_select="@odata.type,AttributeType") + for col_name in columns + ] + requests: List[_RawRequest] = [] + for col_name, attr_meta in zip(columns, attr_metas): + if not attr_meta or not attr_meta.get("MetadataId"): + raise MetadataError( + f"Column '{col_name}' not found on table '{op.table}'.", + subcode=METADATA_COLUMN_NOT_FOUND, + ) + requests.append(self._od._build_delete_column(metadata_id, attr_meta["MetadataId"])) + return requests + + # ------------------------------------------------------------------ + # Query resolvers — delegate to _AsyncODataClient._build_* methods + # ------------------------------------------------------------------ + + async def _resolve_query_sql(self, op: _QuerySql) -> List[_RawRequest]: + return [await self._od._build_sql(op.sql)] diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_odata.py b/src/PowerPlatform/Dataverse/aio/data/_async_odata.py new file mode 100644 index 00000000..08bd9327 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_odata.py @@ -0,0 +1,1835 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async Dataverse Web API client with CRUD, SQL query, and table/column metadata management.""" + +from __future__ import annotations + +__all__ = [] + +import asyncio +import json +import time +import warnings +from datetime import datetime, timezone +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List, Optional, Union + +if TYPE_CHECKING: + import aiohttp + +from urllib.parse import quote as _url_quote + +from ..core._async_http import _AsyncHttpClient, _AsyncResponse +from ._async_upload import _AsyncFileUploadMixin +from ._async_relationships import _AsyncRelationshipOperationsMixin +from ...core.errors import * +from ...data._raw_request import _RawRequest +from ...core._error_codes import ( + _http_subcode, + _is_transient_status, + VALIDATION_SQL_NOT_STRING, + VALIDATION_SQL_EMPTY, + VALIDATION_UNSUPPORTED_COLUMN_TYPE, + METADATA_ENTITYSET_NOT_FOUND, + METADATA_ENTITYSET_NAME_MISSING, + METADATA_TABLE_NOT_FOUND, + METADATA_TABLE_ALREADY_EXISTS, + METADATA_COLUMN_NOT_FOUND, +) + +from ...data._odata_base import ( + _ODataBase, + _GUID_RE, + _extract_pagingcookie, + _USER_AGENT, + _DEFAULT_EXPECTED_STATUSES, + _RequestContext, +) + + +class _AsyncODataClient(_AsyncFileUploadMixin, _AsyncRelationshipOperationsMixin, _ODataBase): + """Async Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" + + def __init__( + self, + auth, + base_url: str, + config=None, + session: Optional[aiohttp.ClientSession] = None, + ) -> None: + """Initialize the async OData client. + + Sets up authentication, base URL, configuration, and internal caches. + + :param auth: Async authentication manager providing ``_acquire_token(scope)`` that returns an object with ``access_token``. + :type auth: ~PowerPlatform.Dataverse.aio.core._async_auth._AsyncAuthManager + :param base_url: Organization base URL (e.g. ``"https://.crm.dynamics.com"``). + :type base_url: ``str`` + :param config: Optional Dataverse configuration (HTTP retry, backoff, timeout, language code). If omitted ``DataverseConfig.from_env()`` is used. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig | ``None`` + :param session: ``aiohttp.ClientSession`` for HTTP connection pooling. Must remain open for the lifetime of this client. + :type session: :class:`aiohttp.ClientSession` | ``None`` + :raises ValueError: If ``base_url`` is empty after stripping. + """ + super().__init__(base_url, config) + self.auth = auth + self._http = _AsyncHttpClient( + retries=self.config.http_retries, + backoff=self.config.http_backoff, + timeout=self.config.http_timeout, + session=session, + logger=self._http_logger, + ) + # Prevents concurrent coroutines from racing through the picklist TTL check + # and issuing redundant metadata fetches. + self._picklist_cache_lock: asyncio.Lock = asyncio.Lock() + + async def close(self) -> None: + """Close the async OData client and release resources. + + Clears all internal caches and closes the underlying HTTP client. + Safe to call multiple times. + """ + super().close() # sync: clears caches, closes logger + if self._http is not None: + await self._http.close() + + async def _headers(self) -> Dict[str, str]: + """Build standard OData headers with bearer auth.""" + scope = f"{self.base_url}/.default" + token = (await self.auth._acquire_token(scope)).access_token + ua = f"{_USER_AGENT} ({self._operation_context})" if self._operation_context else _USER_AGENT + return { + "Authorization": f"Bearer {token}", + "Accept": "application/json", + "Content-Type": "application/json", + "OData-MaxVersion": "4.0", + "OData-Version": "4.0", + "User-Agent": ua, + } + + async def _raw_request(self, method: str, url: str, **kwargs) -> _AsyncResponse: + return await self._http._request(method, url, **kwargs) + + async def _request( + self, + method: str, + url: str, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + **kwargs, + ) -> _AsyncResponse: + # Acquire base headers once (async), then use a sync closure for _RequestContext.build. + # _RequestContext.build is a sync classmethod defined in _ODataBase and shared by both + # sync and async clients — keeping it sync avoids duplicating the header-injection logic. + base_headers = await self._headers() + + def _merge(h: Optional[Dict[str, str]]) -> Dict[str, str]: + if not h: + return base_headers.copy() + merged = base_headers.copy() + merged.update(h) + return merged + + request_context = _RequestContext.build( + method, + url, + expected=expected, + merge_headers=_merge, + **kwargs, + ) + + r = await self._raw_request(request_context.method, request_context.url, **request_context.kwargs) + if r.status in request_context.expected: + return r + + response_headers = getattr(r, "headers", {}) or {} + raw_text = r.text + body_excerpt = raw_text[:200] + svc_code = None + msg = f"HTTP {r.status}" + try: + data = json.loads(raw_text) if raw_text else {} + if isinstance(data, dict): + inner = data.get("error") + if isinstance(inner, dict): + svc_code = inner.get("code") + imsg = inner.get("message") + if isinstance(imsg, str) and imsg.strip(): + msg = imsg.strip() + else: + imsg2 = data.get("message") + if isinstance(imsg2, str) and imsg2.strip(): + msg = imsg2.strip() + except Exception: + pass + sc = r.status + subcode = _http_subcode(sc) + request_id = ( + response_headers.get("x-ms-service-request-id") + or response_headers.get("req_id") + or response_headers.get("x-ms-request-id") + ) + traceparent = response_headers.get("traceparent") + ra = response_headers.get("Retry-After") + retry_after = None + if ra: + try: + retry_after = int(ra) + except Exception: + retry_after = None + is_transient = _is_transient_status(sc) + raise HttpError( + msg, + status_code=sc, + subcode=subcode, + service_error_code=svc_code, + correlation_id=request_context.headers.get( + "x-ms-correlation-id" + ), # this is a value set on client side, although it's logged on server side too + client_request_id=request_context.headers.get( + "x-ms-client-request-id" + ), # this is a value set on client side, although it's logged on server side too + service_request_id=request_id, + traceparent=traceparent, + body_excerpt=body_excerpt, + retry_after=retry_after, + is_transient=is_transient, + ) + + async def _execute_raw( + self, + req: _RawRequest, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + ) -> _AsyncResponse: + """Execute a ``_RawRequest`` and return the HTTP response. + + Encodes the pre-serialised body (if present) as UTF-8 and merges any + per-request headers into the standard OData header set. + """ + kwargs: Dict[str, Any] = {} + if req.body is not None: + kwargs["data"] = req.body.encode("utf-8") + if req.headers: + kwargs["headers"] = req.headers + return await self._request(req.method.lower(), req.url, expected=expected, **kwargs) + + # --- CRUD Internal functions --- + async def _create(self, entity_set: str, table_schema_name: str, record: Dict[str, Any]) -> str: + """Create a single record and return its GUID. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param record: Attribute payload mapped by logical column names. + :type record: ``dict[str, Any]`` + + :return: Created record GUID. + :rtype: ``str`` + + .. note:: + Relies on ``OData-EntityId`` (canonical) or ``Location`` response header. No response body parsing is performed. Raises ``RuntimeError`` if neither header contains a GUID. + """ + r = await self._execute_raw(await self._build_create(entity_set, table_schema_name, record)) + ent_loc = r.headers.get("OData-EntityId") or r.headers.get("OData-EntityID") + if ent_loc: + m = _GUID_RE.search(ent_loc) + if m: + return m.group(0) + loc = r.headers.get("Location") + if loc: + m = _GUID_RE.search(loc) + if m: + return m.group(0) + header_keys = ", ".join(sorted(r.headers.keys())) + raise RuntimeError( + f"Create response missing GUID in OData-EntityId/Location headers (status={r.status}). Headers: {header_keys}" + ) + + async def _create_multiple( + self, + entity_set: str, + table_schema_name: str, + records: List[Dict[str, Any]], + ) -> List[str]: + """Create multiple records using the collection-bound ``CreateMultiple`` action. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param records: Payload dictionaries mapped by column schema names. + :type records: ``list[dict[str, Any]]`` + + :return: List of created record GUIDs (may be empty if response lacks IDs). + :rtype: ``list[str]`` + + .. note:: + Logical type stamping: if any payload omits ``@odata.type`` the client injects ``Microsoft.Dynamics.CRM.``. If all payloads already include ``@odata.type`` no modification occurs. + """ + if not all(isinstance(r, dict) for r in records): + raise TypeError("All items for multi-create must be dicts") + r = await self._execute_raw(await self._build_create_multiple(entity_set, table_schema_name, records)) + try: + body = r.json() + except ValueError: + body = {} + if not isinstance(body, dict): + return [] + # Expected: { "Ids": [guid, ...] } + ids = body.get("Ids") + if isinstance(ids, list): + return [i for i in ids if isinstance(i, str)] + + value = body.get("value") + if isinstance(value, list): + # Extract IDs if possible + out: List[str] = [] + for item in value: + if isinstance(item, dict): + # Heuristic: look for a property ending with 'id' + for k, v in item.items(): + if isinstance(k, str) and k.lower().endswith("id") and isinstance(v, str) and len(v) >= 32: + out.append(v) + break + return out + return [] + + async def _upsert( + self, + entity_set: str, + table_schema_name: str, + alternate_key: Dict[str, Any], + record: Dict[str, Any], + ) -> None: + """Upsert a single record using an alternate key. + + Issues a PATCH request to ``{entity_set}({key_pairs})`` where ``key_pairs`` + is the OData alternate key segment built from ``alternate_key``. Creates the + record if it does not exist; updates it if it does. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param alternate_key: Mapping of alternate key attribute names to their values + used to identify the target record in the URL. + :type alternate_key: ``dict[str, Any]`` + :param record: Attribute payload to set on the record. + :type record: ``dict[str, Any]`` + + :return: ``None`` + :rtype: ``None`` + """ + record = self._lowercase_keys(record) + record = await self._convert_labels_to_ints(table_schema_name, record) + key_str = self._build_alternate_key_str(alternate_key) + url = f"{self.api}/{entity_set}({key_str})" + await self._request("patch", url, json=record, expected=(200, 201, 204)) + + async def _upsert_multiple( + self, + entity_set: str, + table_schema_name: str, + alternate_keys: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ) -> None: + """Upsert multiple records using the collection-bound ``UpsertMultiple`` action. + + Each target is formed by merging the corresponding alternate key fields and record + fields. The ``@odata.type`` annotation is injected automatically if absent. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param alternate_keys: List of alternate key dictionaries, one per record. + Order is significant: ``alternate_keys[i]`` must correspond to ``records[i]``. + Python ``list`` preserves insertion order, so the correspondence is guaranteed + as long as both lists are built from the same source in the same order. + :type alternate_keys: ``list[dict[str, Any]]`` + :param records: List of record payload dictionaries, one per record. + Must be the same length as ``alternate_keys``. + :type records: ``list[dict[str, Any]]`` + + :return: ``None`` + :rtype: ``None`` + + :raises ValueError: If ``alternate_keys`` and ``records`` differ in length, or if + any record payload contains an alternate key field with a conflicting value. + """ + if len(alternate_keys) != len(records): + raise ValueError( + f"alternate_keys and records must have the same length " f"({len(alternate_keys)} != {len(records)})" + ) + logical_name = table_schema_name.lower() + lowered_records = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table_schema_name, r) for r in lowered_records] + targets: List[Dict[str, Any]] = [] + for alt_key, record_processed in zip(alternate_keys, converted): + alt_key_lower = self._lowercase_keys(alt_key) + conflicting = { + k for k in set(alt_key_lower) & set(record_processed) if alt_key_lower[k] != record_processed[k] + } + if conflicting: + raise ValueError(f"record payload conflicts with alternate_key on fields: {sorted(conflicting)!r}") + if "@odata.type" not in record_processed: + record_processed["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" + key_str = self._build_alternate_key_str(alt_key) + record_processed["@odata.id"] = f"{entity_set}({key_str})" + targets.append(record_processed) + payload = {"Targets": targets} + url = f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpsertMultiple" + await self._request("post", url, json=payload, expected=(200, 201, 204)) + + # --- Derived helpers for high-level client ergonomics --- + async def _primary_id_attr(self, table_schema_name: str) -> str: + """Return primary key attribute using metadata; error if unavailable.""" + cache_key = self._normalize_cache_key(table_schema_name) + pid = self._logical_primaryid_cache.get(cache_key) + if pid: + return pid + # Resolve metadata (populates _logical_primaryid_cache or raises if table_schema_name unknown) + await self._entity_set_from_schema_name(table_schema_name) + pid2 = self._logical_primaryid_cache.get(cache_key) + if pid2: + return pid2 + raise RuntimeError( + f"PrimaryIdAttribute not resolved for table_schema_name '{table_schema_name}'. Metadata did not include PrimaryIdAttribute." + ) + + async def _update_by_ids( + self, + table_schema_name: str, + ids: List[str], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """Update many records by GUID list using the collection-bound ``UpdateMultiple`` action. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param ids: GUIDs of target records. + :type ids: ``list[str]`` + :param changes: Broadcast patch (``dict``) applied to all IDs, or list of per-record patches (1:1 with ``ids``). + :type changes: ``dict`` | ``list[dict]`` + + :return: ``None`` + :rtype: ``None`` + """ + if not isinstance(ids, list): + raise TypeError("ids must be list[str]") + if not ids: + return None + pk_attr = await self._primary_id_attr(table_schema_name) + entity_set = await self._entity_set_from_schema_name(table_schema_name) + if isinstance(changes, dict): + batch = [{pk_attr: rid, **changes} for rid in ids] + await self._update_multiple(entity_set, table_schema_name, batch) + return + if not isinstance(changes, list): + raise TypeError("changes must be dict or list[dict]") + if len(changes) != len(ids): + raise ValueError("Length of changes list must match length of ids list") + batch: List[Dict[str, Any]] = [] + for rid, patch in zip(ids, changes): + if not isinstance(patch, dict): + raise TypeError("Each patch must be a dict") + batch.append({pk_attr: rid, **patch}) + await self._update_multiple(entity_set, table_schema_name, batch) + + async def _delete_multiple( + self, + table_schema_name: str, + ids: List[str], + ) -> Optional[str]: + """Delete many records by GUID list via the ``BulkDelete`` action. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param ids: GUIDs of records to delete. + :type ids: ``list[str]`` + + :return: BulkDelete asynchronous job identifier when executed in bulk; ``None`` if no IDs provided or single deletes performed. + :rtype: ``str`` | ``None`` + """ + targets = [rid for rid in ids if rid] + if not targets: + return None + response = await self._execute_raw( + await self._build_delete_multiple(table_schema_name, targets), + expected=(200, 202, 204), + ) + job_id = None + try: + body = response.json() + except ValueError: + body = {} + if isinstance(body, dict): + job_id = body.get("JobId") + return job_id + + async def _update(self, table_schema_name: str, key: str, data: Dict[str, Any]) -> None: + """Update an existing record by GUID. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses). + :type key: ``str`` + :param data: Partial entity payload (attributes to patch). + :type data: ``dict[str, Any]`` + :return: ``None`` + :rtype: ``None`` + """ + await self._execute_raw(await self._build_update(table_schema_name, key, data)) + + async def _update_multiple( + self, + entity_set: str, + table_schema_name: str, + records: List[Dict[str, Any]], + ) -> None: + """Bulk update existing records via the collection-bound ``UpdateMultiple`` action. + + :param entity_set: Resolved entity set (plural) name. + :type entity_set: ``str`` + :param table_schema_name: Schema name of the table, e.g. "new_MyTestTable". + :type table_schema_name: ``str`` + :param records: List of patch dictionaries. Each must include the true primary key attribute (e.g. ``accountid``) and one or more fields to update. + :type records: ``list[dict[str, Any]]`` + :return: ``None`` + :rtype: ``None`` + + .. note:: + - Endpoint: ``POST /{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple`` with body ``{"Targets": [...]}``. + - Transactional semantics: if any individual update fails, the entire request rolls back. + - Response content is ignored; no stable contract for returned IDs/representations. + - Caller must supply the correct primary key attribute (e.g. ``accountid``) in every record. + """ + if not isinstance(records, list) or not records or not all(isinstance(r, dict) for r in records): + raise TypeError("records must be a non-empty list[dict]") + await self._execute_raw(await self._build_update_multiple_from_records(entity_set, table_schema_name, records)) + + async def _delete(self, table_schema_name: str, key: str) -> None: + """Delete a record by GUID. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses) + :type key: ``str`` + + :return: ``None`` + :rtype: ``None`` + """ + await self._execute_raw(await self._build_delete(table_schema_name, key)) + + async def _get( + self, + table_schema_name: str, + key: str, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Dict[str, Any]: + """Retrieve a single record. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key: Record GUID (with or without parentheses). + :type key: ``str`` + :param select: Columns to select; joined with commas into $select. + :type select: ``list[str]`` | ``None`` + :param expand: Navigation properties to expand (``$expand``); passed as-is (case-sensitive). + :type expand: ``list[str]`` | ``None`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: ``str`` | ``None`` + + :return: Retrieved record dictionary (may be empty if no selected attributes). + :rtype: ``dict[str, Any]`` + """ + r = await self._execute_raw( + await self._build_get( + table_schema_name, key, select=select, expand=expand, include_annotations=include_annotations + ) + ) + return r.json() + + async def _get_multiple( + self, + table_schema_name: str, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> AsyncIterator[List[Dict[str, Any]]]: + """Iterate records from an entity set, yielding one page (list of dicts) at a time. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param select: Columns to include (``$select``) or ``None``. Column names are automatically lowercased. + :type select: ``list[str]`` | ``None`` + :param filter: OData ``$filter`` expression or ``None``. This is passed as-is without transformation. Users must provide lowercase logical column names (e.g., "statecode eq 0"). + :type filter: ``str`` | ``None`` + :param orderby: Order expressions (``$orderby``) or ``None``. Column names are automatically lowercased. + :type orderby: ``list[str]`` | ``None`` + :param top: Max total records (applied on first request as ``$top``) or ``None``. + :type top: ``int`` | ``None`` + :param expand: Navigation properties to expand (``$expand``) or ``None``. These are case-sensitive and passed as-is. Users must provide exact navigation property names from entity metadata. + :type expand: ``list[str]`` | ``None`` + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: ``int`` | ``None`` + :param count: If ``True``, adds ``$count=true`` to include a total record count in the response. + :type count: ``bool`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: ``str`` | ``None`` + + :return: Async iterator yielding pages (each page is a ``list`` of record dicts). + :rtype: ``AsyncIterator[list[dict[str, Any]]]`` + """ + extra_headers: Dict[str, str] = {} + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + if prefer_parts: + extra_headers["Prefer"] = ",".join(prefer_parts) + + async def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[str, Any]: + headers = extra_headers if extra_headers else None + r = await self._request("get", url, headers=headers, params=params) + try: + return r.json() + except ValueError: + return {} + + entity_set = await self._entity_set_from_schema_name(table_schema_name) + base_url = f"{self.api}/{entity_set}" + params: Dict[str, Any] = {} + if select: + # Lowercase column names for case-insensitive matching + params["$select"] = ",".join(self._lowercase_list(select)) + if filter: + # Filter is passed as-is; users must use lowercase column names in filter expressions + params["$filter"] = filter + if orderby: + # Lowercase column names for case-insensitive matching + params["$orderby"] = ",".join(self._lowercase_list(orderby)) + if expand: + # Lowercase navigation property names for case-insensitive matching + params["$expand"] = ",".join(expand) + if top is not None: + params["$top"] = int(top) + if count: + params["$count"] = "true" + + data = await _do_request(base_url, params=params) + items = data.get("value") if isinstance(data, dict) else None + if isinstance(items, list) and items: + yield [x for x in items if isinstance(x, dict)] + + next_link = None + if isinstance(data, dict): + next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") + + while next_link: + data = await _do_request(next_link) + items = data.get("value") if isinstance(data, dict) else None + if isinstance(items, list) and items: + yield [x for x in items if isinstance(x, dict)] + next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") if isinstance(data, dict) else None + + # --------------------------- SQL Custom API ------------------------- + async def _query_sql(self, sql: str) -> list[dict[str, Any]]: + """Execute a read-only SQL SELECT using the Dataverse Web API ``?sql=`` capability. + + :param sql: Single SELECT statement within the supported subset. + :type sql: ``str`` + + :return: Result rows (empty list if none). + :rtype: ``list[dict[str, Any]]`` + + :raises ValidationError: If ``sql`` is not a ``str`` or is empty. + :raises MetadataError: If logical table name resolution fails. + + .. note:: + Endpoint form: ``GET /{entity_set}?sql=``. The client + extracts the logical table name, resolves the entity set (metadata + cached), then issues the request. ``SELECT *`` raises + :class:`~PowerPlatform.Dataverse.core.errors.ValidationError` -- + it is deliberately rejected, not silently rewritten. + """ + if not isinstance(sql, str): + raise ValidationError("sql must be a string", subcode=VALIDATION_SQL_NOT_STRING) + if not sql.strip(): + raise ValidationError("sql must be a non-empty string", subcode=VALIDATION_SQL_EMPTY) + sql = sql.strip() + + # Apply safety guardrails (block unsupported syntax including writes, + # warn on risky patterns). SELECT * raises ValidationError here before + # any table resolution. + sql = self._sql_guardrails(sql) + + r = await self._execute_raw(await self._build_sql(sql)) + try: + body = r.json() + except ValueError: + return [] + + # Collect first page + results: list[dict[str, Any]] = [] + if isinstance(body, list): + return [row for row in body if isinstance(row, dict)] + if not isinstance(body, dict): + return results + + value = body.get("value") + if isinstance(value, list): + results = [row for row in value if isinstance(row, dict)] + + # Follow pagination links until exhausted + raw_link = body.get("@odata.nextLink") or body.get("odata.nextLink") + next_link: str | None = raw_link if isinstance(raw_link, str) else None + visited: set[str] = set() + seen_cookies: set[str] = set() + while next_link: + # Guard 1: exact URL cycle (same next_link returned twice) + if next_link in visited: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + "the Dataverse server returned the same nextLink URL twice, " + "indicating an infinite pagination cycle. " + "Returning the rows collected so far. " + "To avoid pagination entirely, add a TOP clause to your query.", + RuntimeWarning, + stacklevel=4, + ) + break + visited.add(next_link) + # Guard 2: server-side bug where pagingcookie does not advance between + # pages (pagenumber increments but cookie GUIDs stay the same), which + # causes an infinite loop even though URLs differ. + cookie = _extract_pagingcookie(next_link) + if cookie is not None: + if cookie in seen_cookies: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + "the Dataverse server returned the same pagingcookie twice " + "(pagenumber incremented but the paging position did not advance). " + "This is a server-side bug. Returning the rows collected so far. " + "To avoid pagination entirely, add a TOP clause to your query.", + RuntimeWarning, + stacklevel=4, + ) + break + seen_cookies.add(cookie) + try: + page_resp = await self._request("get", next_link) + except Exception as exc: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + f"the next-page request failed: {exc}. " + "Add a TOP clause to your query to limit results to a single page.", + RuntimeWarning, + stacklevel=5, + ) + break + try: + page_body = page_resp.json() + except ValueError as exc: + warnings.warn( + f"SQL pagination stopped after {len(results)} rows — " + f"the next-page response was not valid JSON: {exc}. " + "Add a TOP clause to your query to limit results to a single page.", + RuntimeWarning, + stacklevel=5, + ) + break + if not isinstance(page_body, dict): + break + page_value = page_body.get("value") + if not isinstance(page_value, list) or not page_value: + break + results.extend(row for row in page_value if isinstance(row, dict)) + raw_link = page_body.get("@odata.nextLink") or page_body.get("odata.nextLink") + next_link = raw_link if isinstance(raw_link, str) else None + + return results + + # ---------------------- Entity set resolution ----------------------- + async def _entity_set_from_schema_name(self, table_schema_name: str) -> str: + """Resolve entity set name (plural) from a schema name (singular) name using metadata. + + Caches results for subsequent queries. Case-insensitive. + """ + if not table_schema_name: + raise ValueError("table schema name required") + + # Use normalized (lowercase) key for cache lookup + cache_key = self._normalize_cache_key(table_schema_name) + cached = self._logical_to_entityset_cache.get(cache_key) + if cached: + return cached + url = f"{self.api}/EntityDefinitions" + # LogicalName in Dataverse is stored in lowercase, so we need to lowercase for the filter + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + params = { + "$select": "LogicalName,EntitySetName,PrimaryIdAttribute", + "$filter": f"LogicalName eq '{logical_escaped}'", + } + r = await self._request("get", url, params=params) + try: + body = r.json() + items = body.get("value", []) if isinstance(body, dict) else [] + except ValueError: + items = [] + if not items: + plural_hint = ( + " (did you pass a plural entity set name instead of the singular table schema name?)" + if table_schema_name.endswith("s") and not table_schema_name.endswith("ss") + else "" + ) + raise MetadataError( + f"Unable to resolve entity set for table schema name '{table_schema_name}'. Provide the singular table schema name.{plural_hint}", + subcode=METADATA_ENTITYSET_NOT_FOUND, + ) + md = items[0] + es = md.get("EntitySetName") + if not es: + raise MetadataError( + f"Metadata response missing EntitySetName for table schema name '{table_schema_name}'.", + subcode=METADATA_ENTITYSET_NAME_MISSING, + ) + self._logical_to_entityset_cache[cache_key] = es + primary_id_attr = md.get("PrimaryIdAttribute") + if isinstance(primary_id_attr, str) and primary_id_attr: + self._logical_primaryid_cache[cache_key] = primary_id_attr + return es + + # ---------------------- Table metadata helpers ---------------------- + async def _get_entity_by_table_schema_name( + self, + table_schema_name: str, + headers: Optional[Dict[str, str]] = None, + ) -> Optional[Dict[str, Any]]: + """Get entity metadata by table schema name. Case-insensitive. + + Note: LogicalName is stored lowercase in Dataverse, so we lowercase the input + for case-insensitive matching. The response includes SchemaName, LogicalName, + EntitySetName, and MetadataId. + """ + url = f"{self.api}/EntityDefinitions" + # LogicalName is stored lowercase, so we lowercase the input for lookup + logical_lower = table_schema_name.lower() + logical_escaped = self._escape_odata_quotes(logical_lower) + params = { + "$select": "MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute", + "$filter": f"LogicalName eq '{logical_escaped}'", + } + r = await self._request("get", url, params=params, headers=headers) + items = (r.json()).get("value", []) + return items[0] if items else None + + async def _create_entity( + self, + table_schema_name: str, + display_name: str, + attributes: List[Dict[str, Any]], + solution_unique_name: Optional[str] = None, + ) -> Dict[str, Any]: + url = f"{self.api}/EntityDefinitions" + payload = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": table_schema_name, + "DisplayName": self._label(display_name), + "DisplayCollectionName": self._label(display_name + "s"), + "Description": self._label(f"Custom entity for {display_name}"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsActivity": False, + "Attributes": attributes, + } + params = None + if solution_unique_name: + params = {"SolutionUniqueName": solution_unique_name} + await self._request("post", url, json=payload, params=params) + ent = await self._get_entity_by_table_schema_name( + table_schema_name, + headers={"Consistency": "Strong"}, + ) + if not ent or not ent.get("EntitySetName"): + raise RuntimeError( + f"Failed to create or retrieve entity '{table_schema_name}' (EntitySetName not available)." + ) + if not ent.get("MetadataId"): + raise RuntimeError(f"MetadataId missing after creating entity '{table_schema_name}'.") + return ent + + async def _get_attribute_metadata( + self, + entity_metadata_id: str, + column_name: str, + extra_select: Optional[str] = None, + ) -> Optional[Dict[str, Any]]: + # Convert to lowercase logical name for lookup + logical_name = column_name.lower() + attr_escaped = self._escape_odata_quotes(logical_name) + url = f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes" + select_fields = ["MetadataId", "LogicalName", "SchemaName"] + if extra_select: + for piece in extra_select.split(","): + piece = piece.strip() + if not piece or piece in select_fields: + continue + if piece.startswith("@"): + continue + if piece not in select_fields: + select_fields.append(piece) + params = { + "$select": ",".join(select_fields), + "$filter": f"LogicalName eq '{attr_escaped}'", + } + r = await self._request("get", url, params=params) + try: + body = r.json() + except ValueError: + return None + items = body.get("value") if isinstance(body, dict) else None + if isinstance(items, list) and items: + item = items[0] + if isinstance(item, dict): + return item + return None + + async def _list_columns( + self, + table_schema_name: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """List all attribute (column) definitions for a table. + + Issues ``GET EntityDefinitions({MetadataId})/Attributes`` with optional + ``$select`` and ``$filter`` query parameters. + + :param table_schema_name: Schema name of the table + (e.g. ``"account"`` or ``"new_Product"``). + :type table_schema_name: ``str`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + :param filter: Optional OData ``$filter`` expression. For example, + ``"AttributeType eq 'String'"`` returns only string columns. + :type filter: ``str`` or ``None`` + + :return: List of raw attribute metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table is not found. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + metadata_id = ent["MetadataId"] + url = f"{self.api}/EntityDefinitions({metadata_id})/Attributes" + params: Dict[str, str] = {} + if select: + params["$select"] = ",".join(select) + if filter: + params["$filter"] = filter + r = await self._request("get", url, params=params) + return (r.json()).get("value", []) + + async def _wait_for_attribute_visibility( + self, + entity_set: str, + attribute_name: str, + delays: tuple = (0, 3, 10, 20), + ) -> None: + """Wait for a newly created attribute to become visible in the data API. + + After creating an attribute via the metadata API, there can be a delay before + it becomes queryable in the data API. This method polls the entity set with + the attribute in the $select clause until it succeeds or all delays are exhausted. + """ + # Convert to lowercase logical name for URL + logical_name = attribute_name.lower() + probe_url = f"{self.api}/{entity_set}?$top=1&$select={logical_name}" + last_error = None + total_wait = sum(delays) + + for delay in delays: + if delay: + await asyncio.sleep(delay) + try: + await self._request("get", probe_url) + return + except Exception as ex: + last_error = ex + continue + + # All retries exhausted - raise with context + raise RuntimeError( + f"Attribute '{logical_name}' did not become visible in the data API " + f"after {total_wait} seconds (exhausted all retries)." + ) from last_error + + async def _request_metadata_with_retry(self, method: str, url: str, **kwargs) -> _AsyncResponse: + """Fetch metadata with retries on transient errors.""" + max_attempts = 5 + backoff_seconds = 0.4 + for attempt in range(1, max_attempts + 1): + try: + return await self._request(method, url, **kwargs) + except HttpError as err: + if getattr(err, "status_code", None) == 404: + if attempt < max_attempts: + await asyncio.sleep(backoff_seconds * (2 ** (attempt - 1))) + continue + raise RuntimeError(f"Metadata request failed after {max_attempts} retries (404): {url}") from err + raise + + async def _bulk_fetch_picklists(self, table_schema_name: str) -> None: + """Fetch all picklist attributes and their options for a table in one API call. + + Uses collection-level PicklistAttributeMetadata cast to retrieve every picklist + attribute on the table, including its OptionSet options. Populates the nested + cache so that ``_convert_labels_to_ints`` resolves labels without further API calls. + The Dataverse metadata API does not page results. + """ + table_key = self._normalize_cache_key(table_schema_name) + + # Fast path: skip the lock when the cache is already warm. + now = time.time() + table_entry = self._picklist_label_cache.get(table_key) + if isinstance(table_entry, dict) and (now - table_entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: + return + + # Slow path: acquire the lock so that only one coroutine issues the metadata + # fetch. The second TTL check inside the lock handles the case where another + # coroutine populated the cache while we were waiting. + async with self._picklist_cache_lock: + now = time.time() + table_entry = self._picklist_label_cache.get(table_key) + if isinstance(table_entry, dict) and (now - table_entry.get("ts", 0)) < self._picklist_cache_ttl_seconds: + return + + table_esc = self._escape_odata_quotes(table_schema_name.lower()) + url = ( + f"{self.api}/EntityDefinitions(LogicalName='{table_esc}')" + f"/Attributes/Microsoft.Dynamics.CRM.PicklistAttributeMetadata" + f"?$select=LogicalName&$expand=OptionSet($select=Options)" + ) + response = await self._request_metadata_with_retry("get", url) + body = response.json() + items = body.get("value", []) if isinstance(body, dict) else [] + + picklists: Dict[str, Dict[str, int]] = {} + for item in items: + if not isinstance(item, dict): + continue + ln = item.get("LogicalName", "").lower() + if not ln: + continue + option_set = item.get("OptionSet") or {} + options = option_set.get("Options") if isinstance(option_set, dict) else None + mapping: Dict[str, int] = {} + if isinstance(options, list): + for opt in options: + if not isinstance(opt, dict): + continue + val = opt.get("Value") + if not isinstance(val, int): + continue + label_def = opt.get("Label") or {} + locs = label_def.get("LocalizedLabels") + if isinstance(locs, list): + for loc in locs: + if isinstance(loc, dict): + lab = loc.get("Label") + if isinstance(lab, str) and lab.strip(): + normalized = self._normalize_picklist_label(lab) + mapping.setdefault(normalized, val) + picklists[ln] = mapping + + self._picklist_label_cache[table_key] = {"ts": now, "picklists": picklists} + + async def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any]) -> Dict[str, Any]: + """Return a copy of record with any labels converted to option ints. + + Heuristic: For each string value, attempt to resolve against picklist metadata. + If attribute isn't a picklist or label not found, value left unchanged. + + On first encounter of a table, bulk-fetches all picklist attributes and + their options in a single API call, then resolves labels from the warm cache. + """ + resolved_record = record.copy() + + # Check if there are any string-valued candidates worth resolving + has_candidates = any( + isinstance(v, str) and v.strip() and isinstance(k, str) and "@odata." not in k + for k, v in resolved_record.items() + ) + if not has_candidates: + return resolved_record + + # Bulk-fetch all picklists for this table (1 API call, cached for TTL) + await self._bulk_fetch_picklists(table_schema_name) + + # Resolve labels from the nested cache + table_key = self._normalize_cache_key(table_schema_name) + table_entry = self._picklist_label_cache.get(table_key) + if not isinstance(table_entry, dict): + return resolved_record + picklists = table_entry.get("picklists", {}) + + for k, v in resolved_record.items(): + if not isinstance(v, str) or not v.strip(): + continue + if isinstance(k, str) and "@odata." in k: + continue + attr_key = self._normalize_cache_key(k) + mapping = picklists.get(attr_key) + if not isinstance(mapping, dict) or not mapping: + continue + norm = self._normalize_picklist_label(v) + val = mapping.get(norm) + if val is not None: + resolved_record[k] = val + return resolved_record + + async def _get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: + """Return basic metadata for a custom table if it exists. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: Metadata summary or ``None`` if not found. + :rtype: ``dict[str, Any]`` | ``None`` + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent: + return None + return { + "table_schema_name": ent.get("SchemaName") or table_schema_name, + "table_logical_name": ent.get("LogicalName"), + "entity_set_name": ent.get("EntitySetName"), + "metadata_id": ent.get("MetadataId"), + "primary_name_attribute": ent.get("PrimaryNameAttribute"), + "primary_id_attribute": ent.get("PrimaryIdAttribute"), + "columns_created": [], + } + + async def _list_tables( + self, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all non-private tables (``IsPrivate eq false``). + + :param filter: Optional additional OData ``$filter`` expression that is + combined with the default ``IsPrivate eq false`` clause using + ``and``. For example, ``"SchemaName eq 'Account'"`` becomes + ``"IsPrivate eq false and (SchemaName eq 'Account')"``. + When ``None`` (the default), only the ``IsPrivate eq false`` filter + is applied. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase) because + ``EntityDefinitions`` uses PascalCase property names. + When ``None`` (the default) or an empty list, no ``$select`` is + applied and all properties are returned. Passing a bare string + raises ``TypeError``. + :type select: ``list[str]`` or ``None`` + + :return: Metadata entries for non-private tables (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises HttpError: If the metadata request fails. + """ + r = await self._execute_raw(self._build_list_entities(filter=filter, select=select)) + return (r.json()).get("value", []) + + async def _delete_table(self, table_schema_name: str) -> None: + """Delete a table by schema name. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: ``None`` + :rtype: ``None`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the delete request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + await self._execute_raw(self._build_delete_entity(ent["MetadataId"])) + + # ------------------- Alternate key metadata helpers ------------------- + + async def _create_alternate_key( + self, + table_schema_name: str, + key_name: str, + columns: List[str], + display_name_label=None, + ) -> Dict[str, Any]: + """Create an alternate key on a table. + + Issues ``POST EntityDefinitions(LogicalName='{logical_name}')/Keys`` + with ``EntityKeyMetadata`` payload. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key_name: Schema name for the new alternate key. + :type key_name: ``str`` + :param columns: List of column logical names that compose the key. + :type columns: ``list[str]`` + :param display_name_label: Label for the key display name. + :type display_name_label: ``Label`` or ``None`` + + :return: Dictionary with ``metadata_id``, ``schema_name``, and ``key_attributes``. + :rtype: ``dict[str, Any]`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys" + payload: Dict[str, Any] = { + "SchemaName": key_name, + "KeyAttributes": columns, + } + if display_name_label is not None: + payload["DisplayName"] = display_name_label.to_dict() + r = await self._request("post", url, json=payload) + metadata_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "metadata_id": metadata_id, + "schema_name": key_name, + "key_attributes": columns, + } + + async def _get_alternate_keys(self, table_schema_name: str) -> List[Dict[str, Any]]: + """List all alternate keys on a table. + + Issues ``GET EntityDefinitions(LogicalName='{logical_name}')/Keys``. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + + :return: List of raw ``EntityKeyMetadata`` dictionaries. + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys" + r = await self._request("get", url) + return (r.json()).get("value", []) + + async def _delete_alternate_key(self, table_schema_name: str, key_id: str) -> None: + """Delete an alternate key by metadata ID. + + Issues ``DELETE EntityDefinitions(LogicalName='{logical_name}')/Keys({key_id})``. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param key_id: Metadata GUID of the alternate key. + :type key_id: ``str`` + + :return: ``None`` + :rtype: ``None`` + + :raises MetadataError: If the table does not exist. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + logical_name = ent.get("LogicalName", table_schema_name.lower()) + url = f"{self.api}/EntityDefinitions(LogicalName='{logical_name}')/Keys({key_id})" + await self._request("delete", url) + + async def _create_table( + self, + table_schema_name: str, + schema: Dict[str, Any], + solution_unique_name: Optional[str] = None, + primary_column_schema_name: Optional[str] = None, + display_name: Optional[str] = None, + ) -> Dict[str, Any]: + """Create a custom table with specified columns. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param schema: Mapping of column name -> type spec (``str`` or ``Enum`` subclass). + :type schema: ``dict[str, Any]`` + :param solution_unique_name: Optional solution container for the new table; if provided must be non-empty. + :type solution_unique_name: ``str`` | ``None`` + :param primary_column_schema_name: Optional primary column schema name. + :type primary_column_schema_name: ``str`` | ``None`` + :param display_name: Human-readable display name for the table. Defaults to ``table_schema_name``. + :type display_name: ``str`` | ``None`` + + :return: Metadata summary for the created table including created column schema names. + :rtype: ``dict[str, Any]`` + + :raises MetadataError: If the table already exists. + :raises ValueError: If a column type is unsupported or ``solution_unique_name`` is empty. + :raises TypeError: If ``solution_unique_name`` is not a ``str`` when provided. + :raises HttpError: If underlying HTTP requests fail. + """ + # Check if table already exists (case-insensitive) + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if ent: + raise MetadataError( + f"Table '{table_schema_name}' already exists.", + subcode=METADATA_TABLE_ALREADY_EXISTS, + ) + + created_cols: List[str] = [] + + # Use provided primary column name, or derive from table_schema_name prefix (e.g., "new_Product" -> "new_Name"). + # If no prefix detected, default to "new_Name"; server will validate overall table schema. + if primary_column_schema_name: + primary_attr_schema = primary_column_schema_name + else: + primary_attr_schema = ( + f"{table_schema_name.split('_',1)[0]}_Name" if "_" in table_schema_name else "new_Name" + ) + + attributes: List[Dict[str, Any]] = [] + attributes.append(self._attribute_payload(primary_attr_schema, "string", is_primary_name=True)) + for col_name, dtype in schema.items(): + payload = self._attribute_payload(col_name, dtype) + if not payload: + raise ValueError(f"Unsupported column type '{dtype}' for '{col_name}'.") + attributes.append(payload) + created_cols.append(col_name) + + if solution_unique_name is not None: + if not isinstance(solution_unique_name, str): + raise TypeError("solution_unique_name must be a string when provided") + if not solution_unique_name: + raise ValueError("solution_unique_name cannot be empty") + + if display_name is not None: + if not isinstance(display_name, str) or not display_name.strip(): + raise TypeError("display_name must be a non-empty string when provided") + + metadata = await self._create_entity( + table_schema_name=table_schema_name, + display_name=display_name if display_name is not None else table_schema_name, + attributes=attributes, + solution_unique_name=solution_unique_name, + ) + + return { + "table_schema_name": table_schema_name, + "table_logical_name": metadata.get("LogicalName"), + "entity_set_name": metadata.get("EntitySetName"), + "metadata_id": metadata.get("MetadataId"), + "primary_name_attribute": metadata.get("PrimaryNameAttribute"), + "primary_id_attribute": metadata.get("PrimaryIdAttribute"), + "columns_created": created_cols, + } + + async def _create_columns( + self, + table_schema_name: str, + columns: Dict[str, Any], + ) -> List[str]: + """Create new columns on an existing table. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param columns: Mapping of column schema name -> type spec (``str`` or ``Enum`` subclass). + :type columns: ``dict[str, Any]`` + + :return: List of created column schema names. + :rtype: ``list[str]`` + + :raises TypeError: If ``columns`` is not a non-empty dict. + :raises MetadataError: If the target table does not exist. + :raises ValueError: If a column type is unsupported. + :raises HttpError: If an underlying HTTP request fails. + """ + if not isinstance(columns, dict) or not columns: + raise TypeError("columns must be a non-empty dict[name -> type]") + + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + metadata_id = ent.get("MetadataId") + created: List[str] = [] + needs_picklist_flush = False + + for column_name, column_type in columns.items(): + attr = self._attribute_payload(column_name, column_type) + if not attr: + raise ValidationError( + f"Unsupported column type '{column_type}' for column '{column_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + if "OptionSet" in attr: + needs_picklist_flush = True + req = _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + await self._execute_raw(req) + created.append(column_name) + + if needs_picklist_flush: + self._flush_cache("picklist") + + return created + + async def _delete_columns( + self, + table_schema_name: str, + columns: Union[str, List[str]], + ) -> List[str]: + """Delete one or more columns from a table. + + :param table_schema_name: Schema name of the table. + :type table_schema_name: ``str`` + :param columns: Single column name or list of column names + :type columns: ``str`` | ``list[str]`` + + :return: List of deleted column schema names (empty if none removed). + :rtype: ``list[str]`` + + :raises TypeError: If ``columns`` is neither a ``str`` nor ``list[str]``. + :raises ValueError: If any provided column name is empty. + :raises MetadataError: If the table or a specified column does not exist. + :raises RuntimeError: If column metadata lacks a required ``MetadataId``. + :raises HttpError: If an underlying delete request fails. + """ + if isinstance(columns, str): + names = [columns] + elif isinstance(columns, list): + names = columns + else: + raise TypeError("columns must be str or list[str]") + + for name in names: + if not isinstance(name, str) or not name.strip(): + raise ValueError("column names must be non-empty strings") + + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + # Use the actual SchemaName from the entity metadata + entity_schema = ent.get("SchemaName") or table_schema_name + metadata_id = ent.get("MetadataId") + deleted: List[str] = [] + needs_picklist_flush = False + + attr_metas = [ + await self._get_attribute_metadata(metadata_id, col, extra_select="@odata.type,AttributeType") + for col in names + ] + for column_name, attr_meta in zip(names, attr_metas): + if not attr_meta: + raise MetadataError( + f"Column '{column_name}' not found on table '{entity_schema}'.", + subcode=METADATA_COLUMN_NOT_FOUND, + ) + + attr_metadata_id = attr_meta.get("MetadataId") + if not attr_metadata_id: + raise RuntimeError(f"Metadata incomplete for column '{column_name}' (missing MetadataId).") + + await self._execute_raw(self._build_delete_column(metadata_id, attr_metadata_id)) + + attr_type = attr_meta.get("@odata.type") or attr_meta.get("AttributeType") + if isinstance(attr_type, str): + attr_type_l = attr_type.lower() + if "picklist" in attr_type_l or "optionset" in attr_type_l: + needs_picklist_flush = True + + deleted.append(column_name) + + if needs_picklist_flush: + self._flush_cache("picklist") + + return deleted + + # ---------------------- _build_* methods (no HTTP, but may call async helpers) --------------- + + async def _build_create( + self, + entity_set: str, + table: str, + data: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record POST request without sending it.""" + body = self._lowercase_keys(data) + body = await self._convert_labels_to_ints(table, body) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}", + body=json.dumps(body, ensure_ascii=False), + content_id=content_id, + ) + + async def _build_create_multiple( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build a CreateMultiple POST request without sending it.""" + if not all(isinstance(r, dict) for r in records): + raise TypeError("All items for multi-create must be dicts") + logical_name = table.lower() + lowered = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered] + enriched = [ + {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} if "@odata.type" not in r else r + for r in converted + ] + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.CreateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + async def _build_update( + self, + table: str, + record_id: str, + changes: Dict[str, Any], + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record PATCH request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference; in that case the + URL is the reference itself (resolved server-side within a changeset). + """ + body = self._lowercase_keys(changes) + body = await self._convert_labels_to_ints(table, body) + if record_id.startswith("$"): + url = record_id + else: + entity_set = await self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + headers={"If-Match": "*"}, + content_id=content_id, + ) + + async def _build_update_multiple_from_records( + self, + entity_set: str, + table: str, + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request from pre-assembled records. + + Each record must already contain the primary key attribute. This helper + is shared by :meth:`_update_multiple` (which pre-assembles records) and + :meth:`_build_update_multiple` (which assembles from ids + changes). + """ + logical_name = table.lower() + lowered = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered] + enriched = [ + {**r, "@odata.type": f"Microsoft.Dynamics.CRM.{logical_name}"} if "@odata.type" not in r else r + for r in converted + ] + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpdateMultiple", + body=json.dumps({"Targets": enriched}, ensure_ascii=False), + ) + + async def _build_update_multiple( + self, + entity_set: str, + table: str, + ids: List[str], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> _RawRequest: + """Build an UpdateMultiple POST request without sending it.""" + pk_attr = await self._primary_id_attr(table) + if isinstance(changes, dict): + records = [{pk_attr: rid, **changes} for rid in ids] + elif isinstance(changes, list): + if len(changes) != len(ids): + raise ValidationError( + "ids and changes lists must have equal length for paired update.", + subcode="ids_changes_length_mismatch", + ) + records = [{pk_attr: rid, **ch} for rid, ch in zip(ids, changes)] + else: + raise ValidationError("changes must be a dict or list[dict].", subcode="invalid_changes_type") + return await self._build_update_multiple_from_records(entity_set, table, records) + + async def _build_upsert( + self, + entity_set: str, + table: str, + alternate_key: Dict[str, Any], + record: Dict[str, Any], + ) -> _RawRequest: + """Build a single-record PATCH upsert request without sending it. + + Unlike :meth:`_build_update`, no ``If-Match: *`` header is added so the + server creates the record when it does not yet exist. + """ + body = self._lowercase_keys(record) + body = await self._convert_labels_to_ints(table, body) + key_str = self._build_alternate_key_str(alternate_key) + url = f"{self.api}/{entity_set}({key_str})" + return _RawRequest( + method="PATCH", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + async def _build_upsert_multiple( + self, + entity_set: str, + table: str, + alternate_keys: List[Dict[str, Any]], + records: List[Dict[str, Any]], + ) -> _RawRequest: + """Build an UpsertMultiple POST request without sending it.""" + if len(alternate_keys) != len(records): + raise ValidationError( + f"alternate_keys and records must have the same length " f"({len(alternate_keys)} != {len(records)})", + subcode="upsert_length_mismatch", + ) + logical_name = table.lower() + lowered_records = [self._lowercase_keys(r) for r in records] + converted = [await self._convert_labels_to_ints(table, r) for r in lowered_records] + targets: List[Dict[str, Any]] = [] + for alt_key, record_processed in zip(alternate_keys, converted): + alt_key_lower = self._lowercase_keys(alt_key) + conflicting = { + k for k in set(alt_key_lower) & set(record_processed) if alt_key_lower[k] != record_processed[k] + } + if conflicting: + raise ValidationError( + f"record payload conflicts with alternate_key on fields: {sorted(conflicting)!r}", + subcode="upsert_key_conflict", + ) + if "@odata.type" not in record_processed: + record_processed["@odata.type"] = f"Microsoft.Dynamics.CRM.{logical_name}" + key_str = self._build_alternate_key_str(alt_key) + record_processed["@odata.id"] = f"{entity_set}({key_str})" + targets.append(record_processed) + return _RawRequest( + method="POST", + url=f"{self.api}/{entity_set}/Microsoft.Dynamics.CRM.UpsertMultiple", + body=json.dumps({"Targets": targets}, ensure_ascii=False), + ) + + async def _build_delete( + self, + table: str, + record_id: str, + *, + content_id: Optional[int] = None, + ) -> _RawRequest: + """Build a single-record DELETE request without sending it. + + ``record_id`` may be a ``"$n"`` content-ID reference. + """ + if record_id.startswith("$"): + url = record_id + else: + entity_set = await self._entity_set_from_schema_name(table) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + return _RawRequest( + method="DELETE", + url=url, + headers={"If-Match": "*"}, + content_id=content_id, + ) + + async def _build_delete_multiple(self, table: str, ids: List[str]) -> _RawRequest: + """Build a BulkDelete POST request without sending it.""" + pk_attr = await self._primary_id_attr(table) + logical_name = table.lower() + timestamp = datetime.now(timezone.utc).isoformat(timespec="seconds").replace("+00:00", "Z") + payload = { + "JobName": f"Bulk delete {table} records @ {timestamp}", + "SendEmailNotification": False, + "ToRecipients": [], + "CCRecipients": [], + "RecurrencePattern": "", + "StartDateTime": timestamp, + "QuerySet": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.QueryExpression", + "EntityName": logical_name, + "ColumnSet": { + "@odata.type": "Microsoft.Dynamics.CRM.ColumnSet", + "AllColumns": False, + "Columns": [], + }, + "Criteria": { + "@odata.type": "Microsoft.Dynamics.CRM.FilterExpression", + "FilterOperator": "And", + "Conditions": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.ConditionExpression", + "AttributeName": pk_attr, + "Operator": "In", + "Values": [{"Value": rid, "Type": "System.Guid"} for rid in ids], + } + ], + }, + } + ], + } + return _RawRequest( + method="POST", + url=f"{self.api}/BulkDelete", + body=json.dumps(payload, ensure_ascii=False), + ) + + async def _build_get( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> _RawRequest: + """Build a single-record GET request without sending it.""" + entity_set = await self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if expand: + params.append("$expand=" + ",".join(expand)) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if params: + url += "?" + "&".join(params) + headers = None + if include_annotations: + headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} + return _RawRequest(method="GET", url=url, headers=headers) + + async def _build_list( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> _RawRequest: + """Build a multi-record GET request (single page, no pagination) without sending it.""" + entity_set = await self._entity_set_from_schema_name(table) + params: List[str] = [] + if select: + params.append("$select=" + ",".join(self._lowercase_list(select))) + if filter: + params.append("$filter=" + filter) + if orderby: + params.append("$orderby=" + ",".join(orderby)) + if top is not None: + params.append(f"$top={top}") + if expand: + params.append("$expand=" + ",".join(expand)) + if count: + params.append("$count=true") + url = f"{self.api}/{entity_set}" + if params: + url += "?" + "&".join(params) + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None + return _RawRequest(method="GET", url=url, headers=headers) + + async def _build_sql(self, sql: str) -> _RawRequest: + """Build a SQL query GET request without sending it. + + Resolves the entity set from the table name in the SQL statement via + :meth:`_extract_logical_table`, then embeds the SQL as a URL-encoded + ``?sql=`` query parameter. + + Uses ``urllib.parse.quote`` (``%20`` for spaces) rather than + ``urllib.parse.urlencode`` (``+`` for spaces). Both are accepted by + Dataverse and ``%20`` is the canonical RFC 3986 encoding for query- + string values. + + :param sql: SELECT statement (non-empty string; caller is responsible + for validation). + """ + logical = self._extract_logical_table(sql) + entity_set = await self._entity_set_from_schema_name(logical) + return _RawRequest( + method="GET", + url=f"{self.api}/{entity_set}?sql={_url_quote(sql, safe='')}", + ) diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py b/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py new file mode 100644 index 00000000..cbeac29a --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_relationships.py @@ -0,0 +1,263 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async relationship metadata operations for Dataverse Web API. + +This module provides mixin functionality for relationship CRUD operations. +""" + +from __future__ import annotations + +__all__ = [] + +import asyncio +import re +from typing import Any, Dict, List, Optional + +from ...core.errors import MetadataError +from ...core._error_codes import METADATA_TABLE_NOT_FOUND + + +class _AsyncRelationshipOperationsMixin: + """ + Async mixin providing relationship metadata operations. + + This mixin is designed to be used with _AsyncODataClient and depends on: + - self.api: The API base URL + - self._headers(): Method to get auth headers + - self._request(): Async method to make HTTP requests + """ + + async def _create_one_to_many_relationship( + self, + lookup, + relationship, + solution: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Create a one-to-many relationship with lookup attribute. + + Posts to /RelationshipDefinitions with OneToManyRelationshipMetadata. + + :param lookup: Lookup attribute metadata (LookupAttributeMetadata instance). + :type lookup: ~PowerPlatform.Dataverse.models.relationship.LookupAttributeMetadata + :param relationship: Relationship metadata (OneToManyRelationshipMetadata instance). + :type relationship: ~PowerPlatform.Dataverse.models.relationship.OneToManyRelationshipMetadata + :param solution: Optional solution unique name to add the relationship to. + :type solution: ``str`` | ``None`` + + :return: Dictionary with relationship_id, attribute_id, and schema names. + :rtype: ``dict[str, Any]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + + # Build the payload by combining relationship and lookup metadata + payload = relationship.to_dict() + payload["Lookup"] = lookup.to_dict() + + headers = (await self._headers()).copy() + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + + r = await self._request("post", url, headers=headers, json=payload) + + # Extract IDs from response headers + relationship_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "relationship_id": relationship_id, + "relationship_schema_name": relationship.schema_name, + "lookup_schema_name": lookup.schema_name, + "referenced_entity": relationship.referenced_entity, + "referencing_entity": relationship.referencing_entity, + } + + async def _create_many_to_many_relationship( + self, + relationship, + solution: Optional[str] = None, + ) -> Dict[str, Any]: + """ + Create a many-to-many relationship. + + Posts to /RelationshipDefinitions with ManyToManyRelationshipMetadata. + + :param relationship: Relationship metadata (ManyToManyRelationshipMetadata instance). + :type relationship: ~PowerPlatform.Dataverse.models.relationship.ManyToManyRelationshipMetadata + :param solution: Optional solution unique name to add the relationship to. + :type solution: ``str`` | ``None`` + + :return: Dictionary with relationship_id and schema name. + :rtype: ``dict[str, Any]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + + payload = relationship.to_dict() + + headers = (await self._headers()).copy() + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + + r = await self._request("post", url, headers=headers, json=payload) + + # Extract ID from response header + relationship_id = self._extract_id_from_header(r.headers.get("OData-EntityId")) + + return { + "relationship_id": relationship_id, + "relationship_schema_name": relationship.schema_name, + "entity1_logical_name": relationship.entity1_logical_name, + "entity2_logical_name": relationship.entity2_logical_name, + } + + async def _delete_relationship(self, relationship_id: str) -> None: + """ + Delete a relationship by its metadata ID. + + :param relationship_id: The GUID of the relationship metadata. + :type relationship_id: ``str`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions({relationship_id})" + headers = (await self._headers()).copy() + headers["If-Match"] = "*" + await self._request("delete", url, headers=headers) + + async def _get_relationship(self, schema_name: str) -> Optional[Dict[str, Any]]: + """ + Retrieve relationship metadata by schema name. + + :param schema_name: The schema name of the relationship. + :type schema_name: ``str`` + + :return: Relationship metadata dictionary, or None if not found. + :rtype: ``dict[str, Any]`` | ``None`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + params = {"$filter": f"SchemaName eq '{self._escape_odata_quotes(schema_name)}'"} + r = await self._request("get", url, headers=await self._headers(), params=params) + data = r.json() + results = data.get("value", []) + return results[0] if results else None + + async def _list_relationships( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationship definitions. + + Issues ``GET /RelationshipDefinitions`` with optional ``$filter`` and + ``$select`` query parameters. + + :param filter: Optional OData ``$filter`` expression. For example, + ``"RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'"`` + returns only one-to-many relationships. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + + :return: List of raw relationship metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises HttpError: If the Web API request fails. + """ + url = f"{self.api}/RelationshipDefinitions" + params: Dict[str, str] = {} + if filter: + params["$filter"] = filter + if select: + params["$select"] = ",".join(select) + r = await self._request("get", url, headers=await self._headers(), params=params) + return (r.json()).get("value", []) + + async def _list_table_relationships( + self, + table_schema_name: str, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationships for a specific table. + + Issues ``GET EntityDefinitions({MetadataId})/OneToManyRelationships``, + ``GET EntityDefinitions({MetadataId})/ManyToOneRelationships``, and + ``GET EntityDefinitions({MetadataId})/ManyToManyRelationships``, + then combines the results. + + :param table_schema_name: Schema name of the table (e.g. ``"account"``). + :type table_schema_name: ``str`` + :param filter: Optional OData ``$filter`` expression applied to each + sub-request. + :type filter: ``str`` or ``None`` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: ``list[str]`` or ``None`` + + :return: Combined list of one-to-many, many-to-one, and many-to-many + relationship metadata dictionaries (may be empty). + :rtype: ``list[dict[str, Any]]`` + + :raises MetadataError: If the table is not found. + :raises HttpError: If the Web API request fails. + """ + ent = await self._get_entity_by_table_schema_name(table_schema_name) + if not ent or not ent.get("MetadataId"): + raise MetadataError( + f"Table '{table_schema_name}' not found.", + subcode=METADATA_TABLE_NOT_FOUND, + ) + + metadata_id = ent["MetadataId"] + # OneToMany/ManyToOne share the same property surface (ReferencedEntity, + # ReferencingEntity, etc.). ManyToManyRelationshipMetadata has a + # different schema -- it only exposes SchemaName plus Entity1/Entity2 + # fields, not ReferencedEntity or ReferencingEntity. Sending a $select + # that includes those properties to the ManyToMany endpoint causes a + # 400: "Could not find a property named 'ReferencedEntity' on type + # 'ManyToManyRelationshipMetadata'". Use separate param dicts. + one_to_many_params: Dict[str, str] = {} + many_to_many_params: Dict[str, str] = {} + if filter: + one_to_many_params["$filter"] = filter + many_to_many_params["$filter"] = filter + if select: + one_to_many_params["$select"] = ",".join(select) + + one_to_many_url = f"{self.api}/EntityDefinitions({metadata_id})/OneToManyRelationships" + many_to_one_url = f"{self.api}/EntityDefinitions({metadata_id})/ManyToOneRelationships" + many_to_many_url = f"{self.api}/EntityDefinitions({metadata_id})/ManyToManyRelationships" + + headers = await self._headers() + r1, r2, r3 = await asyncio.gather( + self._request("get", one_to_many_url, headers=headers, params=one_to_many_params), + self._request("get", many_to_one_url, headers=headers, params=one_to_many_params), + self._request("get", many_to_many_url, headers=headers, params=many_to_many_params), + ) + + return r1.json().get("value", []) + r2.json().get("value", []) + r3.json().get("value", []) + + def _extract_id_from_header(self, header_value: Optional[str]) -> Optional[str]: + """ + Extract a GUID from an OData-EntityId header value. + + :param header_value: The header value containing a URL with GUID. + :type header_value: ``str`` | ``None`` + + :return: Extracted GUID or None if not found. + :rtype: ``str`` | ``None`` + """ + if not header_value: + return None + match = re.search(r"\(([0-9a-fA-F-]+)\)", header_value) + return match.group(1) if match else None diff --git a/src/PowerPlatform/Dataverse/aio/data/_async_upload.py b/src/PowerPlatform/Dataverse/aio/data/_async_upload.py new file mode 100644 index 00000000..59c36967 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/data/_async_upload.py @@ -0,0 +1,193 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async file upload helpers.""" + +from __future__ import annotations + +import asyncio +import math +from pathlib import Path +from typing import Optional +from urllib.parse import quote + + +class _AsyncFileUploadMixin: + """Async file upload capabilities (small + chunk) with auto selection.""" + + async def _upload_file( + self, + table_schema_name: str, + record_id: str, + file_name_attribute: str, + path: str, + mode: Optional[str] = None, + mime_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file to a Dataverse file column with automatic method selection. + + Parameters + ---------- + table_schema_name : :class:`str` + Table schema name, e.g. "account" or "new_MyTestTable". + record_id : :class:`str` + GUID of the target record. + file_name_attribute : :class:`str` + Schema name of the file column attribute (e.g., "new_Document"). If the column doesn't exist, it will be created. + path : :class:`str` + Local filesystem path to the file. + mode : :class:`str` | None + Upload strategy: "auto" (default), "small", or "chunk". + mime_type : :class:`str` | None + Explicit MIME type. If omitted falls back to application/octet-stream. + if_none_match : :class:`bool` + When True (default) only succeeds if column empty. When False overwrites (If-Match: *). + """ + # Resolve entity set from table schema name + entity_set = await self._entity_set_from_schema_name(table_schema_name) + + # Check if the file column exists, create it if it doesn't + entity_metadata = await self._get_entity_by_table_schema_name(table_schema_name) + if entity_metadata: + metadata_id = entity_metadata.get("MetadataId") + if metadata_id: + attr_metadata = await self._get_attribute_metadata(metadata_id, file_name_attribute) + if not attr_metadata: + # Attribute doesn't exist, create it + await self._create_columns(table_schema_name, {file_name_attribute: "file"}) + # Wait for the attribute to become visible in the data API + # Raises RuntimeError with underlying exception if timeout occurs + await self._wait_for_attribute_visibility(entity_set, file_name_attribute) + + mode = (mode or "auto").lower() + + if mode == "auto": + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + size = p.stat().st_size + mode = "small" if size < 128 * 1024 * 1024 else "chunk" + + # Convert schema name to lowercase logical name for URL usage + logical_name = file_name_attribute.lower() + + if mode == "small": + return await self._upload_file_small( + entity_set, record_id, logical_name, path, content_type=mime_type, if_none_match=if_none_match + ) + if mode == "chunk": + return await self._upload_file_chunk(entity_set, record_id, logical_name, path, if_none_match=if_none_match) + raise ValueError(f"Invalid mode '{mode}'. Use 'auto', 'small', or 'chunk'.") + + async def _upload_file_small( + self, + entity_set: str, + record_id: str, + file_name_attribute: str, + path: str, + content_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file (<128MB) via single PATCH.""" + if not record_id: + raise ValueError("record_id required") + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + size = p.stat().st_size + limit = 128 * 1024 * 1024 + if size > limit: + raise ValueError(f"File size {size} exceeds single-upload limit {limit}; use chunk mode.") + data = await asyncio.to_thread(p.read_bytes) + fname = p.name + key = self._format_key(record_id) + url = f"{self.api}/{entity_set}{key}/{file_name_attribute}" + headers = { + "Content-Type": content_type or "application/octet-stream", + "x-ms-file-name": fname, + } + if if_none_match: + headers["If-None-Match"] = "null" + else: + headers["If-Match"] = "*" + # Single PATCH upload; allow default success codes (includes 204) + await self._request("patch", url, headers=headers, data=data) + + async def _upload_file_chunk( + self, + entity_set: str, + record_id: str, + file_name_attribute: str, + path: str, + if_none_match: bool = True, + ) -> None: + """Stream a local file using Dataverse native chunked PATCH protocol. + 1. Initial PATCH with header x-ms-transfer-mode: chunked (empty body) to start session. + 2. Subsequent PATCH calls to Location URL including sessiontoken with binary body segments and headers. Returns 206 for partial chunks and 204 on final. + + Parameters + ---------- + entity_set : :class:`str` + Target entity set (plural logical name), e.g. "accounts". + record_id : :class:`str` + GUID of the target record. + file_name_attribute : :class:`str` + Logical name of the file column attribute. + path : :class:`str` + Local filesystem path to the file. + if_none_match : :class:`bool` + When True sends ``If-None-Match: null`` to only succeed if the column is currently empty. + Set False to always overwrite (uses ``If-Match: *``). + + Returns + ------- + None + Returns nothing on success. Any failure raises an exception. + """ + if not record_id: + raise ValueError("record_id required") + p = Path(path) + if not p.is_file(): + raise FileNotFoundError(f"File not found: {path}") + total_size = p.stat().st_size + fname = p.name + key = self._format_key(record_id) + init_url = f"{self.api}/{entity_set}{key}/{file_name_attribute}?x-ms-file-name={quote(fname)}" + headers = { + "x-ms-transfer-mode": "chunked", + } + if if_none_match: + headers["If-None-Match"] = "null" + else: + headers["If-Match"] = "*" + r_init = await self._request("patch", init_url, headers=headers, data=b"") + location = r_init.headers.get("Location") or r_init.headers.get("location") + if not location: + raise RuntimeError("Missing Location header with sessiontoken for chunked upload") + rec_hdr = r_init.headers.get("x-ms-chunk-size") or r_init.headers.get("X-MS-CHUNK-SIZE") + try: + recommended_size = int(rec_hdr) if rec_hdr else None + except Exception: # noqa: BLE001 + recommended_size = None + effective_size = recommended_size or (4 * 1024 * 1024) + if effective_size <= 0: + raise ValueError("effective chunk size must be positive") + total_chunks = int(math.ceil(total_size / effective_size)) if total_size else 1 + uploaded_bytes = 0 + with p.open("rb") as fh: + for _ in range(total_chunks): + chunk = await asyncio.to_thread(fh.read, effective_size) + if not chunk: + break + start = uploaded_bytes + end = start + len(chunk) - 1 + c_headers = { + "x-ms-file-name": fname, + "Content-Type": "application/octet-stream", + "Content-Range": f"bytes {start}-{end}/{total_size}", + "Content-Length": str(len(chunk)), + } + # Each chunk returns 206 (partial) or 204 (final). Accept both. + await self._request("patch", location, headers=c_headers, data=chunk, expected=(206, 204)) + uploaded_bytes += len(chunk) diff --git a/src/PowerPlatform/Dataverse/aio/models/__init__.py b/src/PowerPlatform/Dataverse/aio/models/__init__.py new file mode 100644 index 00000000..f4c2e3d0 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async data models and type definitions for the Dataverse SDK. + +Provides async-specific models for Dataverse entities: + +- :class:`~PowerPlatform.Dataverse.aio.models.async_query_builder.AsyncQueryBuilder`: Async fluent query builder. +- :class:`~PowerPlatform.Dataverse.aio.models.async_fetchxml_query.AsyncFetchXmlQuery`: Async FetchXML query. +""" + +__all__ = [] diff --git a/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py b/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py new file mode 100644 index 00000000..c10ca6f6 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/async_fetchxml_query.py @@ -0,0 +1,158 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""AsyncFetchXmlQuery — inert async query object returned by AsyncQueryOperations.fetchxml().""" + +from __future__ import annotations + +import warnings +import xml.etree.ElementTree as _ET +from typing import AsyncIterator, List, TYPE_CHECKING +from urllib.parse import unquote as _url_unquote, quote as _url_quote + +from ...core.errors import ValidationError +from ...models.fetchxml_query import _MAX_URL_LENGTH, _MAX_PAGES, _PREFER_HEADER +from ...models.record import QueryResult, Record + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncFetchXmlQuery"] + + +class AsyncFetchXmlQuery: + """Inert async FetchXML query object. No HTTP request is made until + :meth:`execute` or :meth:`execute_pages` is called. + + Obtained via ``client.query.fetchxml(xml)``. + + :param xml: Stripped, well-formed FetchXML string. + :param entity_name: Entity schema name from the ```` element. + :param client: Parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient`. + """ + + def __init__(self, xml: str, entity_name: str, client: "AsyncDataverseClient") -> None: + self._xml = xml + self._entity_name = entity_name + self._client = client + + async def execute(self) -> QueryResult: + """Execute the FetchXML query and return all results as a :class:`QueryResult`. + + Awaitable — fetches all pages and holds every record in memory before + returning. Use :meth:`execute_pages` when the result set may be large. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + rows = await client.query.fetchxml(xml).execute() + df = rows.to_dataframe() + """ + all_records: List[Record] = [] + async for page in self.execute_pages(): + all_records.extend(page.records) + return QueryResult(all_records) + + async def execute_pages(self) -> AsyncIterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Each iteration fires one HTTP request and yields one page. One-shot — + do not iterate more than once. + + :return: Async iterator of per-page :class:`QueryResult` objects. + :rtype: AsyncIterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + async for page in client.query.fetchxml(xml).execute_pages(): + process(page.to_dataframe()) + """ + current_xml = self._xml + page_num = 1 + page_count = 0 + + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(self._entity_name) + base_url = f"{od.api}/{entity_set}" + + while True: + page_count += 1 + if page_count > _MAX_PAGES: + raise ValidationError( + f"FetchXML paging exceeded {_MAX_PAGES} pages. " + "This may indicate a runaway query or a bug in paging cookie propagation." + ) + + encoded_len = len(base_url) + len("?fetchXml=") + len(_url_quote(current_xml, safe="")) + if encoded_len > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML request URL exceeds {_MAX_URL_LENGTH} characters after encoding. " + "Simplify the query or reduce attributes/conditions." + ) + + r = await od._request( + "get", + base_url, + headers={"Prefer": _PREFER_HEADER}, + params={"fetchXml": current_xml}, + ) + try: + data = r.json() + except Exception: + data = {} + + items = data.get("value") if isinstance(data, dict) else None + page_records: List[Record] = [] + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + page_records.append(Record.from_api_response(self._entity_name, item)) + + yield QueryResult(page_records) + + more_raw = data.get("@Microsoft.Dynamics.CRM.morerecords", False) if isinstance(data, dict) else False + more = more_raw is True or (isinstance(more_raw, str) and more_raw.lower() == "true") + if not more: + break + + raw_cookie = ( + data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" + ) + + _cookie_parse_error = False + if raw_cookie: + try: + cookie_el = _ET.fromstring(raw_cookie) + inner_encoded = cookie_el.get("pagingcookie", "") + if inner_encoded: + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") + continue + except (_ET.ParseError, ValueError) as exc: + warnings.warn( + f"FetchXML paging cookie could not be parsed ({exc}); " "falling back to simple paging.", + UserWarning, + stacklevel=2, + ) + _cookie_parse_error = True + + if not _cookie_parse_error: + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) + page_num += 1 + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py b/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py new file mode 100644 index 00000000..e48b3198 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/models/async_query_builder.py @@ -0,0 +1,141 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""AsyncQueryBuilder — async execution layer over the shared QueryBuilder.""" + +from __future__ import annotations + +from typing import AsyncIterator, List + +from ...models.query_builder import _QueryBuilderBase +from ...models.record import QueryResult, Record + +__all__ = ["AsyncQueryBuilder"] + + +class AsyncQueryBuilder(_QueryBuilderBase): + """Async-capable QueryBuilder. + + Identical fluent interface to :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder` + — all chaining methods (``select``, ``where``, ``order_by``, ``top``, ``page_size``, + ``count``, ``expand``, ``include_annotations``, ``include_formatted_values``) are + inherited unchanged. Only the execution methods are overridden as coroutines. + + Obtained via ``client.query.builder(table)`` on an async client. + + Example:: + + from PowerPlatform.Dataverse.models.filters import col + + result = await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()) + for record in result: + print(record["name"]) + """ + + async def execute(self) -> QueryResult: + """Execute the query and return all results as a :class:`QueryResult`. + + Awaitable — fetches all pages and holds every record in memory before + returning. Use :meth:`execute_pages` for lazy per-page streaming. + + At least one of ``select()``, ``where()``, ``top()``, or + ``page_size()`` must be called first to prevent accidental full-table + scans. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + :raises ValueError: If no scope constraint has been set. + :raises RuntimeError: If the builder was not created via + ``client.query.builder()``. + + Example:: + + result = await (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute()) + for record in result: + print(record["name"]) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute() to prevent accidental full-table scans." + ) + params = self.build() + client = self._query_ops._client + all_records: List[Record] = [] + async with client._scoped_odata() as od: + async for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + all_records.extend(Record.from_api_response(params["table"], row) for row in page) + return QueryResult(all_records) + + async def execute_pages(self) -> AsyncIterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Each iteration triggers one network request. One-shot — do not + iterate more than once. + + At least one of ``select()``, ``where()``, ``top()``, or + ``page_size()`` must be called first to prevent accidental full-table + scans. + + :return: Async iterator of per-page + :class:`~PowerPlatform.Dataverse.models.record.QueryResult` objects. + :rtype: AsyncIterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + :raises ValueError: If no scope constraint has been set. + :raises RuntimeError: If the builder was not created via + ``client.query.builder()``. + + Example:: + + async for page in (client.query.builder("account") + .select("name") + .execute_pages()): + process(page.to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute_pages() to prevent accidental full-table scans." + ) + params = self.build() + client = self._query_ops._client + async with client._scoped_odata() as od: + async for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + yield QueryResult([Record.from_api_response(params["table"], row) for row in page]) diff --git a/src/PowerPlatform/Dataverse/aio/operations/__init__.py b/src/PowerPlatform/Dataverse/aio/operations/__init__.py new file mode 100644 index 00000000..62bb4281 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/__init__.py @@ -0,0 +1,13 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +Async operation namespaces for the Dataverse SDK. + +This module contains the async operation namespace classes that organize +SDK operations into logical groups: records, query, tables, files, and batch. +""" + +from typing import List + +__all__: List[str] = [] diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_batch.py b/src/PowerPlatform/Dataverse/aio/operations/async_batch.py new file mode 100644 index 00000000..3b95fea3 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_batch.py @@ -0,0 +1,174 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async batch operation namespaces for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, List + +from ...data._batch_base import _ChangeSet +from ...operations.batch import ( + BatchDataFrameOperations, + BatchQueryOperations, + BatchRecordOperations, + BatchTableOperations, + ChangeSetRecordOperations, +) +from ..data._async_batch import _AsyncBatchClient +from ...models.batch import BatchResult + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + +__all__ = [ + "AsyncBatchRequest", + "AsyncBatchOperations", + "AsyncChangeSet", +] + + +# --------------------------------------------------------------------------- +# Changeset +# --------------------------------------------------------------------------- + + +class AsyncChangeSet: + """ + A transactional group of single-record write operations. + + All operations succeed or are rolled back together. Use as an async context + manager or call :attr:`records` to add operations directly. + + Do not instantiate directly; use :meth:`AsyncBatchRequest.changeset`. + + Example:: + + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + """ + + def __init__(self, internal: _ChangeSet) -> None: + self._internal = internal + self.records = ChangeSetRecordOperations(internal) + + async def __aenter__(self) -> "AsyncChangeSet": + return self + + async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + return None + + +# --------------------------------------------------------------------------- +# AsyncBatchRequest and AsyncBatchOperations +# --------------------------------------------------------------------------- + + +class AsyncBatchRequest: + """ + Builder for constructing and executing a Dataverse OData ``$batch`` request. + + Obtain via :meth:`AsyncBatchOperations.new` (``client.batch.new()``). Add operations + through :attr:`records`, :attr:`tables`, :attr:`query`, and :attr:`dataframe`, + optionally group writes into a :meth:`changeset`, then call :meth:`execute`. + + Operations are executed sequentially in the order added. The resulting + :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` contains one + :class:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse` per HTTP + request dispatched (some operations expand to multiple requests). + + .. note:: + Maximum 1000 HTTP operations per batch. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Contoso"}) + batch.tables.get("account") + async with batch.changeset() as cs: + ref = cs.records.create("contact", {"firstname": "Alice"}) + cs.records.update("account", account_id, { + "primarycontactid@odata.bind": ref + }) + result = await batch.execute() + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + self._items: List[Any] = [] + self._content_id_counter: List[int] = [1] # shared across all changesets + self.records = BatchRecordOperations(self) + self.tables = BatchTableOperations(self) + self.query = BatchQueryOperations(self) + self.dataframe = BatchDataFrameOperations(self) + + def changeset(self) -> AsyncChangeSet: + """ + Create a new :class:`AsyncChangeSet` attached to this batch. + + The changeset is added to the batch immediately. Operations added to + the returned :class:`AsyncChangeSet` via ``cs.records.*`` execute atomically. + + :returns: A new :class:`AsyncChangeSet` ready to receive operations. + + Example:: + + async with batch.changeset() as cs: + cs.records.create("account", {"name": "ACME"}) + cs.records.create("contact", {"firstname": "Bob"}) + """ + internal = _ChangeSet(_counter=self._content_id_counter) + self._items.append(internal) + return AsyncChangeSet(internal) + + async def execute(self, *, continue_on_error: bool = False) -> BatchResult: + """ + Submit the batch to Dataverse and return all responses. + + :param continue_on_error: When False (default), Dataverse stops at the + first failure and returns that operation's error as a 4xx response. + When True, ``Prefer: odata.continue-on-error`` is sent and all + operations are attempted. + :returns: :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` + with one entry per HTTP operation in submission order. + :raises ValidationError: If the batch exceeds 1000 operations or an + unsupported column type is specified. + :raises MetadataError: If metadata pre-resolution fails (table or + column not found) for ``tables.delete``, ``tables.add_columns``, + or ``tables.remove_columns``. + :raises HttpError: On HTTP-level failures (auth, server error, etc.) + that prevent the batch from executing. + """ + async with self._client._scoped_odata() as od: + return await _AsyncBatchClient(od).execute(self._items, continue_on_error=continue_on_error) + + +class AsyncBatchOperations: + """ + Async namespace for batch operations (``client.batch``). + + Accessed via ``client.batch``. Use :meth:`new` to create an + :class:`AsyncBatchRequest` builder. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + + Example:: + + batch = client.batch.new() + batch.records.create("account", {"name": "Fabrikam"}) + result = await batch.execute() + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + def new(self) -> AsyncBatchRequest: + """ + Create a new empty :class:`AsyncBatchRequest` builder. + + :returns: An empty :class:`AsyncBatchRequest`. + """ + return AsyncBatchRequest(self._client) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py b/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py new file mode 100644 index 00000000..fba7b334 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_dataframe.py @@ -0,0 +1,309 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async DataFrame CRUD operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +import pandas as pd + +from ...utils._pandas import dataframe_to_records + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncDataFrameOperations"] + + +class AsyncDataFrameOperations: + """Async namespace for pandas DataFrame CRUD operations. + + Accessed via ``client.dataframe``. Provides DataFrame-oriented wrappers + around the async record-level CRUD operations. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + import pandas as pd + + async with AsyncDataverseClient(base_url, credential) as client: + + # Query records as a DataFrame via SQL + df = await client.dataframe.sql( + "SELECT TOP 100 name FROM account WHERE statecode = 0" + ) + + # Create records from a DataFrame + new_df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + new_df["accountid"] = await client.dataframe.create("account", new_df) + + # Update records + new_df["telephone1"] = ["555-0100", "555-0200"] + await client.dataframe.update("account", new_df, id_column="accountid") + + # Delete records + await client.dataframe.delete("account", new_df["accountid"]) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # --------------------------------------------------------------------- sql + + async def sql(self, sql: str) -> pd.DataFrame: + """Execute a SQL query and return the results as a pandas DataFrame. + + Delegates to :meth:`~PowerPlatform.Dataverse.aio.operations.async_query.AsyncQueryOperations.sql` + and converts the list of records into a single DataFrame. + + :param sql: Supported SQL SELECT statement. + :type sql: :class:`str` + + :return: DataFrame containing all result rows. Returns an empty + DataFrame when no rows match. + :rtype: ~pandas.DataFrame + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``sql`` is not a string or is empty. + + Example: + SQL query to DataFrame:: + + df = await client.dataframe.sql( + "SELECT TOP 100 name, revenue FROM account " + "WHERE statecode = 0 ORDER BY revenue" + ) + print(f"Got {len(df)} rows") + print(df.head()) + + Aggregate query to DataFrame:: + + df = await client.dataframe.sql( + "SELECT a.name, COUNT(c.contactid) as cnt " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "GROUP BY a.name" + ) + """ + rows = await self._client.query.sql(sql) + if not rows: + return pd.DataFrame() + return pd.DataFrame.from_records([r.data for r in rows]) + + # ----------------------------------------------------------------- create + + async def create( + self, + table: str, + records: pd.DataFrame, + ) -> pd.Series: + """Create records from a pandas DataFrame. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param records: DataFrame where each row is a record to create. + :type records: ~pandas.DataFrame + + :return: Series of created record GUIDs, aligned with the input DataFrame index. + :rtype: ~pandas.Series + + :raises TypeError: If ``records`` is not a pandas DataFrame. + :raises ValueError: If ``records`` is empty or the number of returned + IDs does not match the number of input rows. + + .. tip:: + All rows are sent in a single ``CreateMultiple`` request. For very + large DataFrames, consider splitting into smaller batches to avoid + request timeouts. + + Example: + Create records from a DataFrame:: + + import pandas as pd + + df = pd.DataFrame([ + {"name": "Contoso", "telephone1": "555-0100"}, + {"name": "Fabrikam", "telephone1": "555-0200"}, + ]) + df["accountid"] = await client.dataframe.create("account", df) + """ + if not isinstance(records, pd.DataFrame): + raise TypeError("records must be a pandas DataFrame") + + if records.empty: + raise ValueError("records must be a non-empty DataFrame") + + record_list = dataframe_to_records(records) + + # Detect rows where all values were NaN/None (empty dicts after normalization) + empty_rows = [records.index[i] for i, r in enumerate(record_list) if not r] + if empty_rows: + raise ValueError( + f"Records at index(es) {empty_rows} have no non-null values. " + "All rows must contain at least one field to create." + ) + + ids = await self._client.records.create(table, record_list) + + if len(ids) != len(records): + raise ValueError(f"Server returned {len(ids)} IDs for {len(records)} input rows") + + return pd.Series(ids, index=records.index) + + # ----------------------------------------------------------------- update + + async def update( + self, + table: str, + changes: pd.DataFrame, + id_column: str, + clear_nulls: bool = False, + ) -> None: + """Update records from a pandas DataFrame. + + Each row in the DataFrame represents an update. The ``id_column`` specifies which + column contains the record GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param changes: DataFrame where each row contains a record GUID and the fields to update. + :type changes: ~pandas.DataFrame + :param id_column: Name of the DataFrame column containing record GUIDs. + :type id_column: :class:`str` + :param clear_nulls: When ``False`` (default), missing values (NaN/None) are skipped + (the field is left unchanged on the server). When ``True``, missing values are sent + as ``null`` to Dataverse, clearing the field. Use ``True`` only when you intentionally + want NaN/None values to clear fields. + :type clear_nulls: :class:`bool` + + :raises TypeError: If ``changes`` is not a pandas DataFrame. + :raises ValueError: If ``changes`` is empty, ``id_column`` is not found in the + DataFrame, ``id_column`` contains invalid (non-string, empty, or whitespace-only) + values, or no updatable columns exist besides ``id_column``. + When ``clear_nulls`` is ``False`` (default), rows where all change values + are NaN/None produce empty patches and are silently skipped. If all rows + are skipped, the method returns without making an API call. When + ``clear_nulls`` is ``True``, NaN/None values become explicit nulls, so + rows are never skipped. + + .. tip:: + All rows are sent in a single ``UpdateMultiple`` request (or a + single PATCH for one row). For very large DataFrames, consider + splitting into smaller batches to avoid request timeouts. + + Example: + Update records with different values per row:: + + import pandas as pd + + df = pd.DataFrame([ + {"accountid": "guid-1", "telephone1": "555-0100"}, + {"accountid": "guid-2", "telephone1": "555-0200"}, + ]) + await client.dataframe.update("account", df, id_column="accountid") + + Broadcast the same change to all records:: + + df = pd.DataFrame({"accountid": ["guid-1", "guid-2", "guid-3"]}) + df["websiteurl"] = "https://example.com" + await client.dataframe.update("account", df, id_column="accountid") + + Clear a field by setting clear_nulls=True:: + + df = pd.DataFrame([{"accountid": "guid-1", "websiteurl": None}]) + await client.dataframe.update("account", df, id_column="accountid", clear_nulls=True) + """ + if not isinstance(changes, pd.DataFrame): + raise TypeError("changes must be a pandas DataFrame") + if changes.empty: + raise ValueError("changes must be a non-empty DataFrame") + if id_column not in changes.columns: + raise ValueError(f"id_column '{id_column}' not found in DataFrame columns") + + raw_ids = changes[id_column].tolist() + invalid = [changes.index[i] for i, v in enumerate(raw_ids) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"id_column '{id_column}' contains invalid values at row index(es) {invalid}. " + "All IDs must be non-empty strings." + ) + ids = [v.strip() for v in raw_ids] + + change_columns = [column for column in changes.columns if column != id_column] + if not change_columns: + raise ValueError( + "No columns to update. The DataFrame must contain at least one column besides the id_column." + ) + change_list = dataframe_to_records(changes[change_columns], na_as_null=clear_nulls) + + # Filter out rows where all change values were NaN/None (empty dicts) + paired = [(rid, patch) for rid, patch in zip(ids, change_list) if patch] + if not paired: + return + ids_filtered: List[str] = [p[0] for p in paired] + change_filtered: List[Dict[str, Any]] = [p[1] for p in paired] + + if len(ids_filtered) == 1: + await self._client.records.update(table, ids_filtered[0], change_filtered[0]) + else: + await self._client.records.update(table, ids_filtered, change_filtered) + + # ----------------------------------------------------------------- delete + + async def delete( + self, + table: str, + ids: pd.Series, + use_bulk_delete: bool = True, + ) -> Optional[str]: + """Delete records by passing a pandas Series of GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param ids: Series of record GUIDs to delete. + :type ids: ~pandas.Series + :param use_bulk_delete: When ``True`` (default) and ``ids`` contains multiple values, + execute the BulkDelete action and return its async job identifier. + When ``False`` each record is deleted sequentially. + :type use_bulk_delete: :class:`bool` + + :raises TypeError: If ``ids`` is not a pandas Series. + :raises ValueError: If ``ids`` contains invalid (non-string, empty, or + whitespace-only) values. + + :return: BulkDelete job ID when deleting multiple records via BulkDelete; + ``None`` when deleting a single record, using sequential deletion, or + when ``ids`` is empty. + :rtype: :class:`str` or None + + Example: + Delete records using a Series:: + + import pandas as pd + + ids = pd.Series(["guid-1", "guid-2", "guid-3"]) + await client.dataframe.delete("account", ids) + """ + if not isinstance(ids, pd.Series): + raise TypeError("ids must be a pandas Series") + + raw_list = ids.tolist() + if not raw_list: + return None + + invalid = [ids.index[i] for i, v in enumerate(raw_list) if not isinstance(v, str) or not v.strip()] + if invalid: + raise ValueError( + f"ids Series contains invalid values at index(es) {invalid}. " f"All IDs must be non-empty strings." + ) + id_list = [v.strip() for v in raw_list] + + if len(id_list) == 1: + await self._client.records.delete(table, id_list[0]) + return None + return await self._client.records.delete(table, id_list, use_bulk_delete=use_bulk_delete) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_files.py b/src/PowerPlatform/Dataverse/aio/operations/async_files.py new file mode 100644 index 00000000..da755e62 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_files.py @@ -0,0 +1,113 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async file operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Optional, TYPE_CHECKING + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncFileOperations"] + + +class AsyncFileOperations: + """Async namespace for file operations. + + Accessed via ``client.files``. Provides file upload operations for + Dataverse file columns. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + await client.files.upload( + "account", account_id, "new_Document", "/path/to/file.pdf" + ) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- upload + + async def upload( + self, + table: str, + record_id: str, + file_column: str, + path: str, + *, + mode: Optional[str] = None, + mime_type: Optional[str] = None, + if_none_match: bool = True, + ) -> None: + """Upload a file to a Dataverse file column. + + :param table: Schema name of the table (e.g. ``"account"`` or + ``"new_MyTestTable"``). + :type table: :class:`str` + :param record_id: GUID of the target record. + :type record_id: :class:`str` + :param file_column: Schema name of the file column attribute (e.g., + ``"new_Document"``). If the column doesn't exist, it will be + created automatically. + :type file_column: :class:`str` + :param path: Local filesystem path to the file. The stored filename + will be the basename of this path. + :type path: :class:`str` + :param mode: Upload strategy: ``"auto"`` (default), ``"small"``, or + ``"chunk"``. Auto mode selects small or chunked upload based on + file size. + :type mode: :class:`str` or None + :param mime_type: Explicit MIME type to store with the file (e.g. + ``"application/pdf"``). If not provided, defaults to + ``"application/octet-stream"``. + :type mime_type: :class:`str` or None + :param if_none_match: When True (default), sends + ``If-None-Match: null`` header to only succeed if the column is + currently empty. Set False to always overwrite using + ``If-Match: *``. + :type if_none_match: :class:`bool` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the upload fails or the file column is not empty when + ``if_none_match=True``. + :raises FileNotFoundError: If the specified file path does not exist. + + Example: + Upload a PDF file:: + + await client.files.upload( + "account", + account_id, + "new_Contract", + "/path/to/contract.pdf", + mime_type="application/pdf", + ) + + Upload with auto mode selection:: + + await client.files.upload( + "email", + email_id, + "new_Attachment", + "/path/to/large_file.zip", + ) + """ + async with self._client._scoped_odata() as od: + await od._upload_file( + table, + record_id, + file_column, + path, + mode=mode, + mime_type=mime_type, + if_none_match=if_none_match, + ) diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_query.py b/src/PowerPlatform/Dataverse/aio/operations/async_query.py new file mode 100644 index 00000000..7f378380 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_query.py @@ -0,0 +1,371 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async query operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +import xml.etree.ElementTree as _ET +from typing import Any, Dict, List, TYPE_CHECKING +from urllib.parse import quote as _url_quote + +from ...core.errors import MetadataError, ValidationError +from ..models.async_fetchxml_query import AsyncFetchXmlQuery +from ..models.async_query_builder import AsyncQueryBuilder +from ...models.fetchxml_query import _MAX_URL_LENGTH +from ...models.record import Record + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncQueryOperations"] + + +class AsyncQueryOperations: + """Async namespace for query operations. + + Accessed via ``client.query``. Provides query and search operations + against Dataverse tables. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Fluent query builder (recommended) + from PowerPlatform.Dataverse.models.filters import col + + for record in await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()): + print(record["name"]) + + # SQL query + rows = await client.query.sql("SELECT TOP 10 name FROM account ORDER BY name") + for row in rows: + print(row["name"]) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- builder + + def builder(self, table: str) -> AsyncQueryBuilder: + """Create a fluent async query builder for the specified table. + + Returns an :class:`~PowerPlatform.Dataverse.models.async_query_builder.AsyncQueryBuilder` + that can be chained with filter, select, and order methods, then + executed via ``await .execute()`` or iterated via ``async for`` with + ``.execute_pages()``. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :return: An AsyncQueryBuilder instance bound to this client. + :rtype: ~PowerPlatform.Dataverse.models.async_query_builder.AsyncQueryBuilder + + Example:: + + from PowerPlatform.Dataverse.models.filters import col + + result = await (client.query.builder("account") + .select("name", "revenue") + .where(col("statecode") == 0) + .order_by("revenue", descending=True) + .top(100) + .execute()) + for record in result: + print(record["name"]) + + # Lazy paged iteration + async for page in (client.query.builder("account") + .select("name") + .execute_pages()): + process(page.to_dataframe()) + """ + qb = AsyncQueryBuilder(table) + qb._query_ops = self + return qb + + # --------------------------------------------------------------- fetchxml + + def fetchxml(self, xml: str) -> AsyncFetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery` object. + + No HTTP request is made until + :meth:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery.execute` + or + :meth:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery.execute_pages` + is called on the returned object. + + :param xml: Well-formed FetchXML query string. The root ```` + element determines the entity set endpoint. + :type xml: :class:`str` + :return: Inert async query object. + :rtype: :class:`~PowerPlatform.Dataverse.models.async_fetchxml_query.AsyncFetchXmlQuery` + :raises ValidationError: If the FetchXML is not a string, is empty, or exceeds the URL + length limit when encoded. + :raises ValueError: If the FetchXML is missing a root ```` element or name. + + Example:: + + query = client.query.fetchxml(\"\"\" + + + + + + \"\"\") + + # Eager — all pages collected: + result = await query.execute() + df = result.to_dataframe() + + # Lazy — one page at a time: + async for page in query.execute_pages(): + process(page.to_dataframe()) + """ + if not isinstance(xml, str): + raise ValidationError("xml must be a string") + xml = xml.strip() + if not xml: + raise ValidationError("xml must not be empty") + if len(_url_quote(xml, safe="")) > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML exceeds the Dataverse URL length limit ({_MAX_URL_LENGTH:,} characters) when encoded. " + "Use a $batch POST request to send FetchXML in the request body where the limit is 64 KB." + ) + try: + root_el = _ET.fromstring(xml) + except _ET.ParseError as exc: + raise ValidationError(f"xml is not well-formed: {exc}") from exc + entity_el = root_el.find("entity") + if entity_el is None: + raise ValueError("FetchXML must contain an child element") + entity_name = entity_el.get("name", "") + if not entity_name: + raise ValueError("FetchXML element must have a 'name' attribute") + return AsyncFetchXmlQuery(xml, entity_name, self._client) + + # -------------------------------------------------------------------- sql + + async def sql(self, sql: str) -> List[Record]: + """Execute a read-only SQL query using the Dataverse Web API. + + The Dataverse SQL endpoint supports a broad subset of T-SQL:: + + SELECT / SELECT DISTINCT / SELECT TOP N (0-5000) + FROM table [alias] + INNER JOIN / LEFT JOIN (multi-table, no depth limit) + WHERE (=, !=, >, <, >=, <=, LIKE, IN, NOT IN, IS NULL, + IS NOT NULL, BETWEEN, AND, OR, nested parentheses) + GROUP BY column + ORDER BY column [ASC|DESC] + OFFSET n ROWS FETCH NEXT m ROWS ONLY + COUNT(*), SUM(), AVG(), MIN(), MAX() + + ``SELECT *`` is not supported -- specify column names explicitly. + Use :meth:`sql_columns` to discover available column names for a table. + + Not supported: SELECT *, subqueries, CTE, HAVING, UNION, + RIGHT/FULL/CROSS JOIN, CASE, COALESCE, window functions, + string/date/math functions, INSERT/UPDATE/DELETE. For writes, use + ``client.records`` methods. + + :param sql: Supported SQL SELECT statement. + :type sql: :class:`str` + + :return: List of :class:`~PowerPlatform.Dataverse.models.record.Record` + objects. Returns an empty list when no rows match. + :rtype: list[~PowerPlatform.Dataverse.models.record.Record] + + :raises ~PowerPlatform.Dataverse.core.errors.ValidationError: + If ``sql`` is not a string or is empty. + + Example: + Basic query:: + + rows = await client.query.sql( + "SELECT TOP 10 name FROM account ORDER BY name" + ) + + JOIN with aggregation:: + + rows = await client.query.sql( + "SELECT a.name, COUNT(c.contactid) as cnt " + "FROM account a " + "JOIN contact c ON a.accountid = c.parentcustomerid " + "GROUP BY a.name" + ) + """ + async with self._client._scoped_odata() as od: + rows = await od._query_sql(sql) + return [Record.from_api_response("", row) for row in rows] + + # --------------------------------------------------------------- sql_columns + + async def sql_columns( + self, + table: str, + *, + include_system: bool = False, + ) -> List[Dict[str, Any]]: + """Return a simplified list of SQL-usable columns for a table. + + Each dict contains ``name`` (logical name for SQL), ``type`` + (Dataverse attribute type), ``is_pk`` (primary key flag), and + ``label`` (display name). Virtual columns are always excluded + because the SQL endpoint cannot query them. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param include_system: When ``False`` (default), columns that end + with common system suffixes (``_base``, ``versionnumber``, + ``timezoneruleversionnumber``, ``utcconversiontimezonecode``, + ``importsequencenumber``, ``overriddencreatedon``) are excluded. + :type include_system: :class:`bool` + + :return: List of column metadata dicts. + :rtype: list[dict[str, typing.Any]] + + Example:: + + cols = await client.query.sql_columns("account") + for c in cols: + print(f"{c['name']:30s} {c['type']:20s} PK={c['is_pk']}") + """ + _SYSTEM_SUFFIXES = ( + "_base", + "versionnumber", + "timezoneruleversionnumber", + "utcconversiontimezonecode", + "importsequencenumber", + "overriddencreatedon", + ) + + raw = await self._client.tables.list_columns( + table, + select=[ + "LogicalName", + "SchemaName", + "AttributeType", + "IsPrimaryId", + "IsPrimaryName", + "DisplayName", + "AttributeOf", + ], + filter="AttributeType ne 'Virtual'", + ) + result: List[Dict[str, Any]] = [] + for c in raw: + name = c.get("LogicalName", "") + if not name: + continue + if not include_system and any(name.endswith(s) for s in _SYSTEM_SUFFIXES): + continue + # Skip computed display-name columns (AttributeOf is set, meaning + # they are auto-generated from a lookup column) + if c.get("AttributeOf"): + continue + # Extract display label + label = "" + dn = c.get("DisplayName") + if isinstance(dn, dict): + ul = dn.get("UserLocalizedLabel") + if isinstance(ul, dict): + label = ul.get("Label", "") + result.append( + { + "name": name, + "type": c.get("AttributeType", ""), + "is_pk": bool(c.get("IsPrimaryId")), + "is_name": bool(c.get("IsPrimaryName")), + "label": label, + } + ) + result.sort(key=lambda x: (not x["is_pk"], not x["is_name"], x["name"])) + return result + + # ========================================================================= + # OData helpers -- discover columns, navigation properties, and bind values + # ========================================================================= + + # ------------------------------------------------------- odata_expands + + async def odata_expands( + self, + table: str, + ) -> List[Dict[str, Any]]: + """Discover all ``$expand`` navigation properties from a table. + + Returns entries for each outgoing lookup (single-valued navigation + property). Each entry contains the exact PascalCase navigation + property name needed for ``$expand`` and ``@odata.bind``, plus + the target entity set name. + + :param table: Schema name of the table (e.g. ``"contact"``). + :type table: :class:`str` + + :return: List of dicts, each with: + + - ``nav_property`` -- PascalCase navigation property for $expand + - ``target_table`` -- target entity logical name + - ``target_entity_set`` -- target entity set (for @odata.bind) + - ``lookup_attribute`` -- the lookup column logical name + - ``relationship`` -- relationship schema name + + :rtype: list[dict[str, typing.Any]] + + Example:: + + expands = await client.query.odata_expands("contact") + for e in expands: + print(f"expand={e['nav_property']} -> {e['target_table']}") + + # Use in a query + e = next(e for e in expands if e['target_table'] == 'account') + records = await client.records.list("contact", + select=["fullname"], + expand=[e['nav_property']]) + """ + table_lower = table.lower() + rels = await self._client.tables.list_table_relationships(table) + + result: List[Dict[str, Any]] = [] + for r in rels: + ref_entity = (r.get("ReferencingEntity") or "").lower() + if ref_entity != table_lower: + continue + nav_prop = r.get("ReferencingEntityNavigationPropertyName", "") + target = r.get("ReferencedEntity", "") + lookup_attr = r.get("ReferencingAttribute", "") + schema = r.get("SchemaName", "") + if not nav_prop or not target: + continue + + # Resolve entity set name for @odata.bind + target_set = "" + try: + async with self._client._scoped_odata() as od: + target_set = await od._entity_set_from_schema_name(target) + except (KeyError, AttributeError, ValueError, MetadataError): + pass # Entity set resolution failed; target_set stays empty + + result.append( + { + "nav_property": nav_prop, + "target_table": target, + "target_entity_set": target_set, + "lookup_attribute": lookup_attr, + "relationship": schema, + } + ) + + result.sort(key=lambda x: (x["target_table"], x["nav_property"])) + return result diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_records.py b/src/PowerPlatform/Dataverse/aio/operations/async_records.py new file mode 100644 index 00000000..1a2d86cf --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_records.py @@ -0,0 +1,522 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async record CRUD operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Any, AsyncGenerator, Dict, List, Optional, Union, overload, TYPE_CHECKING + +from ...core.errors import HttpError +from ...models.record import QueryResult, Record +from ...models.upsert import UpsertItem + +if TYPE_CHECKING: + from ...models.filters import FilterExpression + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncRecordOperations"] + + +class AsyncRecordOperations: + """Async namespace for record-level CRUD operations. + + Accessed via ``client.records``. Provides create, update, delete, retrieve, + list, and upsert operations on individual Dataverse records. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Create a single record + guid = await client.records.create("account", {"name": "Contoso Ltd"}) + + # Retrieve a record + record = await client.records.retrieve("account", guid, select=["name"]) + + # Update a record + await client.records.update("account", guid, {"telephone1": "555-0100"}) + + # Delete a record + await client.records.delete("account", guid) + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ------------------------------------------------------------------ create + + @overload + async def create(self, table: str, data: Dict[str, Any]) -> str: ... + + @overload + async def create(self, table: str, data: List[Dict[str, Any]]) -> List[str]: ... + + async def create( + self, + table: str, + data: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> Union[str, List[str]]: + """Create one or more records in a Dataverse table. + + When ``data`` is a single dictionary, creates one record and returns its + GUID as a string. When ``data`` is a list of dictionaries, creates all + records via the ``CreateMultiple`` action and returns a list of GUIDs. + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: :class:`str` + :param data: A single record dictionary or a list of record dictionaries. + Each dictionary maps column schema names to values. + :type data: dict or list[dict] + + :return: A single GUID string for a single record, or a list of GUID + strings for bulk creation. + :rtype: str or list[str] + + :raises TypeError: If ``data`` is not a dict or list[dict]. + + Example: + Create a single record:: + + guid = await client.records.create("account", {"name": "Contoso"}) + print(f"Created: {guid}") + + Create multiple records:: + + guids = await client.records.create("account", [ + {"name": "Contoso"}, + {"name": "Fabrikam"}, + ]) + print(f"Created {len(guids)} accounts") + """ + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(table) + if isinstance(data, dict): + rid = await od._create(entity_set, table, data) + if not isinstance(rid, str): + raise TypeError("_create (single) did not return GUID string") + return rid + if isinstance(data, list): + ids = await od._create_multiple(entity_set, table, data) + if not isinstance(ids, list) or not all(isinstance(x, str) for x in ids): + raise TypeError("_create (multi) did not return list[str]") + return ids + raise TypeError("data must be dict or list[dict]") + + # ------------------------------------------------------------------ update + + async def update( + self, + table: str, + ids: Union[str, List[str]], + changes: Union[Dict[str, Any], List[Dict[str, Any]]], + ) -> None: + """Update one or more records in a Dataverse table. + + Supports three usage patterns: + + 1. **Single** -- ``update("account", "guid", {"name": "New"})`` + 2. **Broadcast** -- ``update("account", [id1, id2], {"status": 1})`` + applies the same changes dict to every ID. + 3. **Paired** -- ``update("account", [id1, id2], [ch1, ch2])`` + applies each changes dict to its corresponding ID (lists must be + equal length). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param ids: A single GUID string, or a list of GUID strings. + :type ids: str or list[str] + :param changes: A dictionary of field changes (single/broadcast), or a + list of dictionaries (paired, one per ID). + :type changes: dict or list[dict] + + :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` + does not match the expected pattern. + + Example: + Single update:: + + await client.records.update("account", account_id, {"telephone1": "555-0100"}) + + Broadcast update:: + + await client.records.update("account", [id1, id2], {"statecode": 1}) + + Paired update:: + + await client.records.update( + "account", + [id1, id2], + [{"name": "Name A"}, {"name": "Name B"}], + ) + """ + async with self._client._scoped_odata() as od: + if isinstance(ids, str): + if not isinstance(changes, dict): + raise TypeError("For single id, changes must be a dict") + await od._update(table, ids, changes) + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + await od._update_by_ids(table, ids, changes) + return None + + # ------------------------------------------------------------------ delete + + @overload + async def delete(self, table: str, ids: str) -> None: ... + + @overload + async def delete(self, table: str, ids: List[str], *, use_bulk_delete: bool = True) -> Optional[str]: ... + + async def delete( + self, + table: str, + ids: Union[str, List[str]], + *, + use_bulk_delete: bool = True, + ) -> Optional[str]: + """Delete one or more records from a Dataverse table. + + When ``ids`` is a single string, deletes that one record. When ``ids`` + is a list, either executes a BulkDelete action (returning the async job + ID) or deletes each record sequentially depending on ``use_bulk_delete``. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param ids: A single GUID string, or a list of GUID strings. + :type ids: str or list[str] + :param use_bulk_delete: When True (default) and ``ids`` is a list, use + the BulkDelete action and return its async job ID. When False, delete + records one at a time. + :type use_bulk_delete: :class:`bool` + + :return: The BulkDelete job ID when bulk-deleting; otherwise None. + :rtype: :class:`str` or None + + :raises TypeError: If ``ids`` is not str or list[str]. + + Example: + Delete a single record:: + + await client.records.delete("account", account_id) + + Bulk delete:: + + job_id = await client.records.delete("account", [id1, id2, id3]) + """ + async with self._client._scoped_odata() as od: + if isinstance(ids, str): + await od._delete(table, ids) + return None + if not isinstance(ids, list): + raise TypeError("ids must be str or list[str]") + if not ids: + return None + if not all(isinstance(rid, str) for rid in ids): + raise TypeError("ids must contain string GUIDs") + if use_bulk_delete: + return await od._delete_multiple(table, ids) + for rid in ids: + await od._delete(table, rid) + return None + + # --------------------------------------------------------------- retrieve + + async def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Optional[Record]: + """Fetch a single record by its GUID, returning ``None`` if not found. + + Returns ``None`` instead of raising when the record does not exist (HTTP 404). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand (e.g. + ``["primarycontactid"]``). Navigation property names are + case-sensitive and must match the entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + :return: Typed record, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None + + Example:: + + record = await client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + if record is not None: + contact = record.get("primarycontactid") or {} + print(contact.get("fullname")) + """ + async with self._client._scoped_odata() as od: + try: + raw = await od._get( + table, record_id, select=select, expand=expand, include_annotations=include_annotations + ) + except HttpError as exc: + if exc.status_code == 404: + return None + raise + return Record.from_api_response(table, raw, record_id=record_id) + + # -------------------------------------------------------------------- list + + async def list( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> QueryResult: + """Fetch multiple records and return them as a :class:`QueryResult`. + + All pages are collected eagerly and returned as a single :class:`QueryResult`. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: All matching records collected into a :class:`QueryResult`. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + from PowerPlatform.Dataverse import col + + result = await client.records.list( + "account", + filter=col("statecode") == 0, + select=["name", "statuscode"], + orderby=["name asc"], + top=100, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + for record in result: + print(record["name"], record.get("statuscode@OData.Community.Display.V1.FormattedValue")) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + all_records: List[Record] = [] + async with self._client._scoped_odata() as od: + async for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + all_records.extend(Record.from_api_response(table, row) for row in page) + return QueryResult(all_records) + + # --------------------------------------------------------------- list_pages + + async def list_pages( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> AsyncGenerator[QueryResult, None]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming counterpart to :meth:`list` — use when you want to process + records page by page without loading all into memory. Each iteration + triggers one network request via ``@odata.nextLink``. One-shot — do + not iterate more than once. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions. + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: Async generator of per-page :class:`QueryResult` objects. + :rtype: AsyncGenerator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`, None] + + Example:: + + async for page in client.records.list_pages( + "account", + filter="statecode eq 0", + orderby=["name asc"], + page_size=200, + ): + process(page.to_dataframe()) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + async with self._client._scoped_odata() as od: + async for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + yield QueryResult([Record.from_api_response(table, row) for row in page]) + + # ------------------------------------------------------------------ upsert + + async def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> None: + """Upsert one or more records identified by alternate keys. + + When ``items`` contains a single entry, performs a single upsert via PATCH + using the alternate key in the URL. When ``items`` contains multiple entries, + uses the ``UpsertMultiple`` bulk action. + + Each item must be either a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + or a plain ``dict`` with ``"alternate_key"`` and ``"record"`` keys (both dicts). + + :param table: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). + :type table: str + :param items: Non-empty list of :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` + instances or dicts with ``"alternate_key"`` and ``"record"`` keys. + :type items: list[UpsertItem | dict] + + :return: ``None`` + :rtype: None + + :raises TypeError: If ``items`` is not a non-empty list, or if any element is + neither a :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem` nor a + dict with ``"alternate_key"`` and ``"record"`` keys. + + Example: + Upsert a single record using ``UpsertItem``:: + + from PowerPlatform.Dataverse.models.upsert import UpsertItem + + await client.records.upsert("account", [ + UpsertItem( + alternate_key={"accountnumber": "ACC-001"}, + record={"name": "Contoso Ltd", "description": "Primary account"}, + ) + ]) + + Upsert a single record using a plain dict:: + + await client.records.upsert("account", [ + { + "alternate_key": {"accountnumber": "ACC-001"}, + "record": {"name": "Contoso Ltd", "description": "Primary account"}, + }, + ]) + + Upsert multiple records using ``UpsertItem``:: + + from PowerPlatform.Dataverse.models.upsert import UpsertItem + + await client.records.upsert("account", [ + UpsertItem( + alternate_key={"accountnumber": "ACC-001"}, + record={"name": "Contoso Ltd", "description": "Primary account"}, + ), + UpsertItem( + alternate_key={"accountnumber": "ACC-002"}, + record={"name": "Fabrikam Inc", "description": "Partner account"}, + ), + ]) + + Upsert multiple records using plain dicts:: + + await client.records.upsert("account", [ + { + "alternate_key": {"accountnumber": "ACC-001"}, + "record": {"name": "Contoso Ltd", "description": "Primary account"}, + }, + { + "alternate_key": {"accountnumber": "ACC-002"}, + "record": {"name": "Fabrikam Inc", "description": "Partner account"}, + }, + ]) + + The ``alternate_key`` dict may contain multiple columns when the configured + alternate key is composite, e.g. + ``{"accountnumber": "ACC-001", "address1_postalcode": "98052"}``. + """ + if not isinstance(items, list) or not items: + raise TypeError("items must be a non-empty list of UpsertItem or dicts") + normalized: List[UpsertItem] = [] + for i in items: + if isinstance(i, UpsertItem): + normalized.append(i) + elif isinstance(i, dict) and isinstance(i.get("alternate_key"), dict) and isinstance(i.get("record"), dict): + normalized.append(UpsertItem(alternate_key=i["alternate_key"], record=i["record"])) + else: + raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") + async with self._client._scoped_odata() as od: + entity_set = await od._entity_set_from_schema_name(table) + if len(normalized) == 1: + item = normalized[0] + await od._upsert(entity_set, table, item.alternate_key, item.record) + else: + alternate_keys = [i.alternate_key for i in normalized] + records = [i.record for i in normalized] + await od._upsert_multiple(entity_set, table, alternate_keys, records) + return None diff --git a/src/PowerPlatform/Dataverse/aio/operations/async_tables.py b/src/PowerPlatform/Dataverse/aio/operations/async_tables.py new file mode 100644 index 00000000..0fbe61c8 --- /dev/null +++ b/src/PowerPlatform/Dataverse/aio/operations/async_tables.py @@ -0,0 +1,838 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Async table metadata operations namespace for the Dataverse SDK.""" + +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Union, TYPE_CHECKING + +from ...models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, + RelationshipInfo, +) +from ...models.table_info import AlternateKeyInfo +from ...models.labels import Label, LocalizedLabel +from ...models.table_info import TableInfo +from ...common.constants import CASCADE_BEHAVIOR_REMOVE_LINK + +if TYPE_CHECKING: + from ..async_client import AsyncDataverseClient + + +__all__ = ["AsyncTableOperations"] + + +class AsyncTableOperations: + """Async namespace for table-level metadata operations. + + Accessed via ``client.tables``. Provides operations to create, delete, + inspect, and list Dataverse tables, as well as add and remove columns. + + :param client: The parent :class:`~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient` instance. + :type client: ~PowerPlatform.Dataverse.aio.async_client.AsyncDataverseClient + + Example:: + + async with AsyncDataverseClient(base_url, credential) as client: + + # Create a table + info = await client.tables.create( + "new_Product", + {"new_Price": "decimal", "new_InStock": "bool"}, + solution="MySolution", + ) + + # List tables + tables = await client.tables.list() + + # Get table info + info = await client.tables.get("new_Product") + + # Add columns + await client.tables.add_columns("new_Product", {"new_Rating": "int"}) + + # Remove columns + await client.tables.remove_columns("new_Product", "new_Rating") + + # Delete a table + await client.tables.delete("new_Product") + """ + + def __init__(self, client: "AsyncDataverseClient") -> None: + self._client = client + + # ----------------------------------------------------------------- create + + async def create( + self, + table: str, + columns: Dict[str, Any], + *, + solution: Optional[str] = None, + primary_column: Optional[str] = None, + display_name: Optional[str] = None, + ) -> TableInfo: + """Create a custom table with the specified columns. + + :param table: Schema name of the table with customization prefix + (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Mapping of column schema names (with customization + prefix) to their types. Supported types include ``"string"`` + (or ``"text"``), ``"memo"`` (or ``"multiline"``), + ``"int"`` (or ``"integer"``), ``"decimal"`` + (or ``"money"``), ``"float"`` (or ``"double"``), ``"datetime"`` + (or ``"date"``), ``"bool"`` (or ``"boolean"``), ``"file"``, and + ``Enum`` subclasses + (for local option sets). + :type columns: :class:`dict` + :param solution: Optional solution unique name that should own the new + table. When omitted the table is created in the default solution. + :type solution: :class:`str` or None + :param primary_column: Optional primary name column schema name with + customization prefix (e.g. ``"new_ProductName"``). If not provided, + defaults to ``"{prefix}_Name"``. + :type primary_column: :class:`str` or None + :param display_name: Human-readable display name for the table + (e.g. ``"Product"``). When omitted, defaults to the table schema name. + :type display_name: :class:`str` or None + + :return: Table metadata with ``schema_name``, ``entity_set_name``, + ``logical_name``, ``metadata_id``, and ``columns_created``. + Supports dict-like access with legacy keys for backward + compatibility. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If table creation fails or the table already exists. + + Example: + Create a table with simple columns:: + + from enum import IntEnum + + class ItemStatus(IntEnum): + ACTIVE = 1 + INACTIVE = 2 + + result = await client.tables.create( + "new_Product", + { + "new_Title": "string", + "new_Price": "decimal", + "new_Status": ItemStatus, + }, + solution="MySolution", + primary_column="new_ProductName", + display_name="Product", + ) + print(f"Created: {result['table_schema_name']}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_table( + table, + columns, + solution, + primary_column, + display_name, + ) + return TableInfo.from_dict(raw) + + # ----------------------------------------------------------------- delete + + async def delete(self, table: str) -> None: + """Delete a custom table by schema name. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist or deletion fails. + + .. warning:: + This operation is irreversible and will delete all records in the + table along with the table definition. + + Example:: + + await client.tables.delete("new_MyTestTable") + """ + async with self._client._scoped_odata() as od: + await od._delete_table(table) + + # -------------------------------------------------------------------- get + + async def get(self, table: str) -> Optional[TableInfo]: + """Get basic metadata for a table if it exists. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"`` + or ``"account"``). + :type table: :class:`str` + + :return: Table metadata, or ``None`` if the table is not found. + Supports dict-like access with legacy keys for backward + compatibility. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.TableInfo` + or None + + Example:: + + info = await client.tables.get("new_MyTestTable") + if info: + print(f"Logical name: {info['table_logical_name']}") + print(f"Entity set: {info['entity_set_name']}") + """ + async with self._client._scoped_odata() as od: + raw = await od._get_table_info(table) + if raw is None: + return None + return TableInfo.from_dict(raw) + + # ------------------------------------------------------------------- list + + async def list( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all non-private tables in the Dataverse environment. + + By default returns every table where ``IsPrivate eq false``. Supply + an optional OData ``$filter`` expression to further narrow the results. + The expression is combined with the default ``IsPrivate eq false`` + clause using ``and``. + + :param filter: Optional OData ``$filter`` expression to further narrow + the list of returned tables (e.g. + ``"SchemaName eq 'Account'"``). Column names in filter + expressions must use the exact property names from the + ``EntityDefinitions`` metadata (typically PascalCase). + :type filter: :class:`str` or None + :param select: Optional list of property names to include in the + response (projected via the OData ``$select`` query option). + Property names must use the exact PascalCase names from the + ``EntityDefinitions`` metadata (e.g. + ``["LogicalName", "SchemaName", "DisplayName"]``). + When ``None`` (the default) or an empty list, all properties are + returned. + :type select: list[str] or None + + :return: List of EntityDefinition metadata dictionaries. + :rtype: list[dict] + + Example:: + + # List all non-private tables + tables = await client.tables.list() + for table in tables: + print(table["LogicalName"]) + + # List only tables whose schema name starts with "new_" + custom_tables = await client.tables.list( + filter="startswith(SchemaName, 'new_')" + ) + + # List tables with only specific properties + tables = await client.tables.list( + select=["LogicalName", "SchemaName", "EntitySetName"] + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_tables(filter=filter, select=select) + + # ------------------------------------------------------------- add_columns + + async def add_columns( + self, + table: str, + columns: Dict[str, Any], + ) -> List[str]: + """Add one or more columns to an existing table. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Mapping of column schema names (with customization + prefix) to their types. Supported types are the same as for + :meth:`create`. + :type columns: :class:`dict` + + :return: Schema names of the columns that were created. + :rtype: list[str] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + + Example:: + + created = await client.tables.add_columns( + "new_MyTestTable", + {"new_Notes": "string", "new_Active": "bool"}, + ) + print(created) # ['new_Notes', 'new_Active'] + """ + async with self._client._scoped_odata() as od: + return await od._create_columns(table, columns) + + # ---------------------------------------------------------- remove_columns + + async def remove_columns( + self, + table: str, + columns: Union[str, List[str]], + ) -> List[str]: + """Remove one or more columns from a table. + + :param table: Schema name of the table (e.g. ``"new_MyTestTable"``). + :type table: :class:`str` + :param columns: Column schema name or list of column schema names to + remove. Must include the customization prefix (e.g. + ``"new_TestColumn"``). + :type columns: str or list[str] + + :return: Schema names of the columns that were removed. + :rtype: list[str] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table or a specified column does not exist. + + Example:: + + removed = await client.tables.remove_columns( + "new_MyTestTable", + ["new_Notes", "new_Active"], + ) + print(removed) # ['new_Notes', 'new_Active'] + """ + async with self._client._scoped_odata() as od: + return await od._delete_columns(table, columns) + + # ------------------------------------------------------ create_one_to_many + + async def create_one_to_many_relationship( + self, + lookup: LookupAttributeMetadata, + relationship: OneToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> RelationshipInfo: + """Create a one-to-many relationship between tables. + + This operation creates both the relationship and the lookup attribute + on the referencing table. + + :param lookup: Metadata defining the lookup attribute. + :type lookup: ~PowerPlatform.Dataverse.models.relationship.LookupAttributeMetadata + :param relationship: Metadata defining the relationship. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.OneToManyRelationshipMetadata + :param solution: Optional solution unique name to add relationship to. + :type solution: :class:`str` or None + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``lookup_schema_name``, ``referenced_entity``, and + ``referencing_entity``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a one-to-many relationship: Department (1) -> Employee (N):: + + from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + Label, + LocalizedLabel, + CascadeConfiguration, + ) + from PowerPlatform.Dataverse.common.constants import ( + CASCADE_BEHAVIOR_REMOVE_LINK, + ) + + lookup = LookupAttributeMetadata( + schema_name="new_DepartmentId", + display_name=Label( + localized_labels=[ + LocalizedLabel(label="Department", language_code=1033) + ] + ), + ) + + relationship = OneToManyRelationshipMetadata( + schema_name="new_Department_Employee", + referenced_entity="new_department", + referencing_entity="new_employee", + referenced_attribute="new_departmentid", + cascade_configuration=CascadeConfiguration( + delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ), + ) + + result = await client.tables.create_one_to_many_relationship(lookup, relationship) + print(f"Created lookup field: {result.lookup_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_one_to_many_relationship( + lookup, + relationship, + solution, + ) + return RelationshipInfo.from_one_to_many( + relationship_id=raw["relationship_id"], + relationship_schema_name=raw["relationship_schema_name"], + lookup_schema_name=raw["lookup_schema_name"], + referenced_entity=raw["referenced_entity"], + referencing_entity=raw["referencing_entity"], + ) + + # ----------------------------------------------------- create_many_to_many + + async def create_many_to_many_relationship( + self, + relationship: ManyToManyRelationshipMetadata, + *, + solution: Optional[str] = None, + ) -> RelationshipInfo: + """Create a many-to-many relationship between tables. + + This operation creates a many-to-many relationship and an intersect + table to manage the relationship. + + :param relationship: Metadata defining the many-to-many relationship. + :type relationship: ~PowerPlatform.Dataverse.models.relationship.ManyToManyRelationshipMetadata + :param solution: Optional solution unique name to add relationship to. + :type solution: :class:`str` or None + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``entity1_logical_name``, and ``entity2_logical_name``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a many-to-many relationship: Employee <-> Project:: + + from PowerPlatform.Dataverse.models.relationship import ( + ManyToManyRelationshipMetadata, + ) + + relationship = ManyToManyRelationshipMetadata( + schema_name="new_employee_project", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + + result = await client.tables.create_many_to_many_relationship(relationship) + print(f"Created: {result.relationship_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._create_many_to_many_relationship( + relationship, + solution, + ) + return RelationshipInfo.from_many_to_many( + relationship_id=raw["relationship_id"], + relationship_schema_name=raw["relationship_schema_name"], + entity1_logical_name=raw["entity1_logical_name"], + entity2_logical_name=raw["entity2_logical_name"], + ) + + # ------------------------------------------------------- delete_relationship + + async def delete_relationship(self, relationship_id: str) -> None: + """Delete a relationship by its metadata ID. + + :param relationship_id: The GUID of the relationship metadata. + :type relationship_id: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + .. warning:: + Deleting a relationship also removes the associated lookup attribute + for one-to-many relationships. This operation is irreversible. + + Example:: + + await client.tables.delete_relationship( + "12345678-1234-1234-1234-123456789abc" + ) + """ + async with self._client._scoped_odata() as od: + await od._delete_relationship(relationship_id) + + # -------------------------------------------------------- get_relationship + + async def get_relationship(self, schema_name: str) -> Optional[RelationshipInfo]: + """Retrieve relationship metadata by schema name. + + :param schema_name: The schema name of the relationship. + :type schema_name: :class:`str` + + :return: Relationship metadata, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + or None + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + rel = await client.tables.get_relationship("new_Department_Employee") + if rel: + print(f"Found: {rel.relationship_schema_name}") + """ + async with self._client._scoped_odata() as od: + raw = await od._get_relationship(schema_name) + if raw is None: + return None + return RelationshipInfo.from_api_response(raw) + + # ------------------------------------------------------- create_lookup_field + + async def create_lookup_field( + self, + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + solution: Optional[str] = None, + language_code: int = 1033, + ) -> RelationshipInfo: + """Create a simple lookup field relationship. + + This is a convenience method that wraps :meth:`create_one_to_many_relationship` + for the common case of adding a lookup field to an existing table. + + :param referencing_table: Logical name of the table that will have + the lookup field (child table). + :type referencing_table: :class:`str` + :param lookup_field_name: Schema name for the lookup field + (e.g., ``"new_AccountId"``). + :type lookup_field_name: :class:`str` + :param referenced_table: Logical name of the table being referenced + (parent table). + :type referenced_table: :class:`str` + :param display_name: Display name for the lookup field. Defaults to + the referenced table name. + :type display_name: :class:`str` or None + :param description: Optional description for the lookup field. + :type description: :class:`str` or None + :param required: Whether the lookup is required. Defaults to ``False``. + :type required: :class:`bool` + :param cascade_delete: Delete behavior (``"RemoveLink"``, + ``"Cascade"``, ``"Restrict"``). Defaults to ``"RemoveLink"``. + :type cascade_delete: :class:`str` + :param solution: Optional solution unique name to add the relationship + to. + :type solution: :class:`str` or None + :param language_code: Language code for labels. Defaults to 1033 + (English). + :type language_code: :class:`int` + + :return: Relationship metadata with ``relationship_id``, + ``relationship_schema_name``, ``relationship_type``, + ``lookup_schema_name``, ``referenced_entity``, and + ``referencing_entity``. + :rtype: :class:`~PowerPlatform.Dataverse.models.relationship.RelationshipInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a simple lookup field:: + + result = await client.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + display_name="Account", + required=True, + cascade_delete=CASCADE_BEHAVIOR_REMOVE_LINK, + ) + print(f"Created lookup: {result['lookup_schema_name']}") + """ + async with self._client._scoped_odata() as od: + lookup, relationship = od._build_lookup_field_models( + referencing_table=referencing_table, + lookup_field_name=lookup_field_name, + referenced_table=referenced_table, + display_name=display_name, + description=description, + required=required, + cascade_delete=cascade_delete, + language_code=language_code, + ) + + return await self.create_one_to_many_relationship(lookup, relationship, solution=solution) + + # ------------------------------------------------- create_alternate_key + + async def create_alternate_key( + self, + table: str, + key_name: str, + columns: List[str], + *, + display_name: Optional[str] = None, + language_code: int = 1033, + ) -> AlternateKeyInfo: + """Create an alternate key on a table. + + Alternate keys allow upsert operations to identify records by one or + more columns instead of the primary GUID. After creation the key is + queued for index building; its :attr:`~AlternateKeyInfo.status` will + transition from ``"Pending"`` to ``"Active"`` once the index is ready. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + :param key_name: Schema name for the new alternate key + (e.g. ``"new_product_code_key"``). + :type key_name: :class:`str` + :param columns: List of column logical names that compose the key + (e.g. ``["new_productcode"]``). + :type columns: list[str] + :param display_name: Display name for the key. Defaults to + ``key_name`` if not provided. + :type display_name: :class:`str` or None + :param language_code: Language code for labels. Defaults to 1033 + (English). + :type language_code: :class:`int` + + :return: Metadata for the newly created alternate key. + :rtype: :class:`~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + Create a single-column alternate key for upsert:: + + key = await client.tables.create_alternate_key( + "new_Product", + "new_product_code_key", + ["new_productcode"], + display_name="Product Code", + ) + print(f"Key ID: {key.metadata_id}") + print(f"Columns: {key.key_attributes}") + """ + label = Label(localized_labels=[LocalizedLabel(label=display_name or key_name, language_code=language_code)]) + async with self._client._scoped_odata() as od: + raw = await od._create_alternate_key(table, key_name, columns, label) + return AlternateKeyInfo( + metadata_id=raw["metadata_id"], + schema_name=raw["schema_name"], + key_attributes=raw["key_attributes"], + status="Pending", + ) + + # --------------------------------------------------- get_alternate_keys + + async def get_alternate_keys(self, table: str) -> List[AlternateKeyInfo]: + """List all alternate keys defined on a table. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + + :return: List of alternate key metadata objects. May be empty if no + alternate keys are defined. + :rtype: list[~PowerPlatform.Dataverse.models.table_info.AlternateKeyInfo] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example: + List alternate keys and print their status:: + + keys = await client.tables.get_alternate_keys("new_Product") + for key in keys: + print(f"{key.schema_name}: {key.status}") + """ + async with self._client._scoped_odata() as od: + raw_list = await od._get_alternate_keys(table) + return [AlternateKeyInfo.from_api_response(item) for item in raw_list] + + # ------------------------------------------------ delete_alternate_key + + async def delete_alternate_key(self, table: str, key_id: str) -> None: + """Delete an alternate key by its metadata ID. + + :param table: Schema name of the table (e.g. ``"new_Product"``). + :type table: :class:`str` + :param key_id: Metadata GUID of the alternate key to delete. + :type key_id: :class:`str` + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table does not exist. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + .. warning:: + Deleting an alternate key that is in use by upsert operations will + cause those operations to fail. This operation is irreversible. + + Example:: + + await client.tables.delete_alternate_key( + "new_Product", + "12345678-1234-1234-1234-123456789abc", + ) + """ + async with self._client._scoped_odata() as od: + await od._delete_alternate_key(table, key_id) + + # -------------------------------------------------------- list_columns + + async def list_columns( + self, + table: str, + *, + select: Optional[List[str]] = None, + filter: Optional[str] = None, + ) -> List[Dict[str, Any]]: + """List all attribute (column) definitions for a table. + + :param table: Schema name of the table (e.g. ``"account"`` or + ``"new_Product"``). + :type table: :class:`str` + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + :param filter: Optional OData ``$filter`` expression. For example, + ``"AttributeType eq 'String'"`` returns only string columns. + :type filter: :class:`str` or None + + :return: List of raw attribute metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table is not found. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all columns on the account table + columns = await client.tables.list_columns("account") + for col in columns: + print(f"{col['LogicalName']} ({col.get('AttributeType')})") + + # List only specific properties + columns = await client.tables.list_columns( + "account", + select=["LogicalName", "SchemaName", "AttributeType"], + ) + + # Filter to only string attributes + columns = await client.tables.list_columns( + "account", + filter="AttributeType eq 'String'", + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_columns(table, select=select, filter=filter) + + # ------------------------------------------------- list_relationships + + async def list_relationships( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationship definitions in the environment. + + :param filter: Optional OData ``$filter`` expression. For example, + ``"RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'"`` + returns only one-to-many relationships. + :type filter: :class:`str` or None + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + + :return: List of raw relationship metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all relationships + rels = await client.tables.list_relationships() + for rel in rels: + print(f"{rel['SchemaName']} ({rel.get('@odata.type')})") + + # Filter by type + one_to_many = await client.tables.list_relationships( + filter="RelationshipType eq Microsoft.Dynamics.CRM.RelationshipType'OneToManyRelationship'" + ) + + # Select specific properties + rels = await client.tables.list_relationships( + select=["SchemaName", "ReferencedEntity", "ReferencingEntity"] + ) + """ + async with self._client._scoped_odata() as od: + return await od._list_relationships(filter=filter, select=select) + + # --------------------------------------------- list_table_relationships + + async def list_table_relationships( + self, + table: str, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> List[Dict[str, Any]]: + """List all relationships for a specific table. + + Combines one-to-many, many-to-one, and many-to-many relationships + for the given table by querying + ``EntityDefinitions({id})/OneToManyRelationships``, + ``EntityDefinitions({id})/ManyToOneRelationships``, and + ``EntityDefinitions({id})/ManyToManyRelationships``. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData ``$filter`` expression applied to each + sub-request. + :type filter: :class:`str` or None + :param select: Optional list of property names to project via + ``$select``. Values are passed as-is (PascalCase). + :type select: list[str] or None + + :return: Combined list of one-to-many, many-to-one, and many-to-many + relationship metadata dictionaries. + :rtype: list[dict[str, typing.Any]] + + :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: + If the table is not found. + :raises ~PowerPlatform.Dataverse.core.errors.HttpError: + If the Web API request fails. + + Example:: + + # List all relationships for the account table + rels = await client.tables.list_table_relationships("account") + for rel in rels: + print(f"{rel['SchemaName']} -> {rel.get('@odata.type')}") + """ + async with self._client._scoped_odata() as od: + return await od._list_table_relationships(table, filter=filter, select=select) diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md index 79d4e342..c3af44c5 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md @@ -20,6 +20,32 @@ This skill provides guidance for developers working on the PowerPlatform Dataver 5. **Consider backwards compatibility** - Avoid breaking changes 6. **Internal vs public naming** - Modules, files, and functions not meant to be part of the public API must use a `_` prefix (e.g., `_odata.py`, `_relationships.py`). Files without the prefix (e.g., `constants.py`, `metadata.py`) are public and importable by SDK consumers +### Dataverse Property Naming Rules + +Dataverse uses two different naming conventions for properties. Getting this wrong causes 400 errors that are hard to debug. + +| Property type | Name convention | Example | When used | +|---|---|---|---| +| **Structural** (columns) | LogicalName (always lowercase) | `new_name`, `new_priority` | `$select`, `$filter`, `$orderby`, record payload keys | +| **Navigation** (relationships / lookups) | Navigation Property Name (usually SchemaName, PascalCase, case-sensitive) | `new_CustomerId`, `new_AgentId` | `$expand`, `@odata.bind` annotation keys | + +Navigation property names are case-sensitive and must match the entity's `$metadata`. Using the logical name instead of the navigation property name results in 400 Bad Request errors. + +**Critical rule:** The OData parser validates `@odata.bind` property names **case-sensitively** against declared navigation properties. Lowercasing `new_CustomerId@odata.bind` to `new_customerid@odata.bind` causes: `ODataException: An undeclared property 'new_customerid' which only has property annotations...` + +**SDK implementation:** + +- `_lowercase_keys()` lowercases all keys EXCEPT those containing `@odata.` (preserves navigation property casing in `@odata.bind` keys) +- `_lowercase_list()` lowercases `$select` and `$orderby` params (structural properties) +- `$expand` params are passed as-is (navigation properties, PascalCase) +- `_convert_labels_to_ints()` skips `@odata.` keys entirely (they are annotations, not attributes) + +**When adding new code that processes record dicts or builds query parameters:** + +- Always use `_lowercase_keys()` for record payloads. Never manually call `.lower()` on all keys +- Never lowercase `$expand` values or `@odata.bind` key prefixes +- If iterating record keys, skip keys containing `@odata.` when doing attribute-level operations + ### Code Style 6. **No emojis** - Do not use emoji in code, comments, or output diff --git a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md index 72677468..d25815d7 100644 --- a/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md +++ b/src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-use/SKILL.md @@ -28,11 +28,23 @@ Use the PowerPlatform Dataverse Client Python SDK to interact with Microsoft Dat The SDK supports Dataverse's native bulk operations: Pass lists to `create()`, `update()` for automatic bulk processing, for `delete()`, set `use_bulk_delete` when passing lists to use bulk operation ### Paging -- Control page size with `page_size` parameter +- Control page size with `page_size` parameter on `records.list()`, `records.list_pages()`, or `QueryBuilder.page_size()` - Use `top` parameter to limit total records returned +- **Preferred**: `client.query.builder(table)....execute_pages()` — composable `where(col(...))` filters, formatted values, expand with nested selects, full pagination control +- Simple streaming shortcut: `records.list_pages(table, *, filter, select, top, orderby, expand, page_size, count, include_annotations)` — string-based OData filter only, yields one `QueryResult` per page +- `execute(by_page=True/False)` is **deprecated** and emits `UserWarning`; use `execute_pages()` instead +- `QueryBuilder.to_dataframe()` is **deprecated**; use `.execute().to_dataframe()` instead + +### QueryResult +- Returned by `records.list()`, `records.retrieve()`, `execute()`, and each page from `list_pages()` / `execute_pages()` +- Iterable: `for record in result` — each item is a `dict`-like `Record` +- `.to_dataframe()` — convert to pandas DataFrame +- `.first()` — return the first record or `None` (safe: returns `None` on empty result) +- `result[n]` — index access returns a `Record`; `result[n:m]` returns a `QueryResult` +- `len(result)` — number of records in this result/page ### DataFrame Support -- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.get()`, `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` +- DataFrame operations are accessed via the `client.dataframe` namespace: `client.dataframe.create()`, `client.dataframe.update()`, `client.dataframe.delete()` — `client.dataframe.get()` is deprecated; use `client.query.builder(table).where(...).execute().to_dataframe()` instead ## Common Operations @@ -85,28 +97,92 @@ contact_ids = client.records.create("contact", contacts) #### Read Records ```python # Get single record by ID -account = client.records.get("account", account_id, select=["name", "telephone1"]) - -# Query with filter (paginated) -for page in client.records.get( - "account", - select=["accountid", "name"], # select is case-insensitive (automatically lowercased) - filter="statecode eq 0", # filter must use lowercase logical names (not transformed) - top=100, -): +account = client.records.retrieve("account", account_id, select=["name", "telephone1"]) + +# With expand — fetch a related record in the same HTTP request +account = client.records.retrieve( + "account", account_id, + select=["name"], + expand=["primarycontactid"], +) +contact = (account.get("primarycontactid") or {}) +print(contact.get("fullname")) + +# Simple shortcut — use records.list() only for basic filter + select without composable logic. +# Follows @odata.nextLink automatically and loads all matching records into memory. +# For filtering, sorting, expansion, or formatted values, prefer client.query.builder() (see below). +result = client.records.list("account", filter="statecode eq 0", select=["name", "accountid"]) +for record in result: + print(record["name"]) +``` + +#### Query Builder (Preferred for Filtering, Sorting, Expand, Formatted Values) + +Use `client.query.builder()` for any query that goes beyond simple filter + select. It provides composable `where(col(...))` expressions, formatted value support, nested expansion, and streaming — all with a fluent API. + +```python +from PowerPlatform.Dataverse.models.filters import col +from PowerPlatform.Dataverse.models.query_builder import ExpandOption + +# Basic query with composable filter and sort +result = (client.query.builder("account") + .select("accountid", "name", "statecode") + .where(col("statecode") == 0) + .order_by("name asc") + .execute()) +for record in result: + print(record["name"]) + +# Composable filters — AND / OR / NOT using Python operators +result = (client.query.builder("contact") + .select("fullname", "emailaddress1") + .where((col("statecode") == 0) & (col("emailaddress1").contains("@contoso.com"))) + .execute()) + +# Formatted values — display labels for option sets, currency symbols, etc. +result = (client.query.builder("account") + .select("accountid", "name", "industrycode") + .where(col("statecode") == 0) + .include_formatted_values() + .execute()) +for record in result: + label = record.get("industrycode@OData.Community.Display.V1.FormattedValue") + print(record["name"], label) + +# Navigation property expansion with nested column select +result = (client.query.builder("account") + .select("name") + .expand(ExpandOption("primarycontactid").select("fullname", "emailaddress1")) + .where(col("statecode") == 0) + .execute()) +for record in result: + contact = record.get("primarycontactid", {}) + print(f"{record['name']} - {contact.get('fullname', 'N/A')}") + +# Stream large result sets page-by-page (memory-efficient) +for page in (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .order_by("name asc") + .page_size(500) + .execute_pages()): for record in page: print(record["name"]) -# Query with navigation property expansion (case-sensitive!) -for page in client.records.get( - "account", - select=["name"], - expand=["primarycontactid"], # Navigation properties are case-sensitive! - filter="statecode eq 0", # Column names must be lowercase logical names -): - for account in page: - contact = account.get("primarycontactid", {}) - print(f"{account['name']} - {contact.get('fullname', 'N/A')}") +# Convert query results to a DataFrame +df = (client.query.builder("account") + .select("accountid", "name") + .where(col("statecode") == 0) + .execute() + .to_dataframe()) + +# Limit total results +result = client.query.builder("account").select("name").top(100).execute() + +# Simple streaming shortcut via records.list_pages() (string filter only, same params as records.list()) +for page in client.records.list_pages("account", filter="statecode eq 0", select=["name"], page_size=500): + for record in page: + print(record["name"]) ``` #### Create Records with Lookup Bindings (@odata.bind) @@ -179,18 +255,24 @@ client.records.delete("account", [id1, id2, id3], use_bulk_delete=True) The SDK provides DataFrame wrappers for all CRUD operations via the `client.dataframe` namespace, using pandas DataFrames and Series as input/output. +> **Note:** `client.dataframe.get()` is deprecated. Use `client.query.builder(table).select(...).where(...).execute().to_dataframe()` instead. `QueryBuilder.to_dataframe()` (without `.execute()`) is also deprecated — always call `.execute()` first. + ```python import pandas as pd -# Query records -- returns a single DataFrame -df = client.dataframe.get("account", filter="statecode eq 0", select=["name"]) +# Query records -- returns a single DataFrame (GA pattern: .execute().to_dataframe()) +from PowerPlatform.Dataverse.models.filters import col +df = client.query.builder("account").where(col("statecode") == 0).select("name").execute().to_dataframe() print(f"Got {len(df)} rows") -# Limit results with top for large tables -df = client.dataframe.get("account", select=["name"], top=100) +# Limit results with top +df = client.query.builder("account").select("name").top(100).execute().to_dataframe() + +# Via records.list() (simpler for basic queries) +df = client.records.list("account", filter="statecode eq 0", select=["name"]).to_dataframe() # Fetch single record as one-row DataFrame -df = client.dataframe.get("account", record_id=account_id, select=["name"]) +df = client.records.retrieve("account", account_id, select=["name"]).to_dataframe() # Create records from a DataFrame (returns a Series of GUIDs) new_accounts = pd.DataFrame([ @@ -223,6 +305,34 @@ for record in results: print(record["name"]) ``` +### FetchXML Queries + +`client.query.fetchxml(xml)` returns an inert `FetchXmlQuery` object — **no HTTP request is made** until `.execute()` or `.execute_pages()` is called. + +```python +xml = """ + + + + + + + + + +""" + +# Load all results into memory (simple, small-to-medium sets) +query = client.query.fetchxml(xml) +result = query.execute() # returns QueryResult — all pages fetched upfront +for record in result: + print(record["name"]) + +# Stream page-by-page (large sets or early exit) +for page in query.execute_pages(): # yields one QueryResult per HTTP page + process(page.to_dataframe()) +``` + ### Table Management #### Create Custom Tables @@ -380,7 +490,8 @@ Use `client.batch` to send multiple operations in one HTTP request. All batch me batch = client.batch.new() batch.records.create("account", {"name": "Contoso"}) batch.records.update("account", account_id, {"telephone1": "555-0100"}) -batch.records.get("account", account_id, select=["name"]) +batch.records.retrieve("account", account_id, select=["name"], expand=["primarycontactid"], include_annotations="OData.Community.Display.V1.FormattedValue") # single record with expand +batch.records.list("account", filter="statecode eq 0", select=["name"], orderby=["name asc"], top=50, page_size=25, count=True) # multi-record, single page batch.query.sql("SELECT TOP 5 name FROM account") result = batch.execute() @@ -412,7 +523,8 @@ print(f"Succeeded: {len(result.succeeded)}, Failed: {len(result.failed)}") **Batch limitations:** - Maximum 1000 operations per batch -- Paginated `records.get()` (without `record_id`) is not supported in batch +- `batch.records.get()` is deprecated; use `batch.records.retrieve()` for single records +- `batch.records.list()` returns a single page (no pagination); use `top` to bound results - `flush_cache()` is not supported in batch ## Error Handling @@ -430,7 +542,7 @@ from PowerPlatform.Dataverse.core.errors import ( from PowerPlatform.Dataverse.client import DataverseClient try: - client.records.get("account", "invalid-id") + client.records.retrieve("account", "invalid-id") except HttpError as e: print(f"HTTP {e.status_code}: {e.message}") print(f"Error code: {e.code}") @@ -464,16 +576,17 @@ except ValidationError as e: ### Performance Optimization -1. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization -2. **Specify select fields** - Limit returned columns to reduce payload size -3. **Control page size** - Use `top` and `page_size` parameters appropriately -4. **Reuse client instances** - Don't create new clients for each operation -5. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations -6. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) -7. **Always include customization prefix** for custom tables/columns -8. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) -9. **Test in non-production environments** first -10. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` +1. **Prefer `client.query.builder()` for any non-trivial query** — use the builder for filtering, sorting, expansion, or formatted values; `records.list()` is a convenience shortcut for simple filter+select only +2. **Use bulk operations** - Pass lists to create/update/delete for automatic optimization +3. **Specify select fields** - Limit returned columns to reduce payload size +4. **Control page size** - Use `top` and `page_size` parameters appropriately; use `execute_pages()` for large sets +5. **Reuse client instances** - Don't create new clients for each operation +6. **Use production credentials** - ClientSecretCredential or CertificateCredential for unattended operations +7. **Error handling** - Implement retry logic for transient errors (`e.is_transient`) +8. **Always include customization prefix** for custom tables/columns +9. **Use lowercase for column names, match `$metadata` for navigation properties** - Column names in `$select`/`$filter`/record payloads use lowercase LogicalNames. Navigation properties in `$expand` and `@odata.bind` keys are case-sensitive and must match the entity's `$metadata` (PascalCase for custom lookups like `new_CustomerId`, lowercase for system lookups like `parentaccountid`) +10. **Test in non-production environments** first +11. **Use named constants** - Import cascade behavior constants from `PowerPlatform.Dataverse.common.constants` ## Additional Resources @@ -486,9 +599,10 @@ Load these resources as needed during development: ## Key Reminders -1. **Schema names are required** - Never use display names -2. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") -3. **Filter is case-sensitive** - Use lowercase logical names -4. **Bulk operations are encouraged** - Pass lists for optimization -5. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` -6. **Structured errors** - Check `is_transient` for retry logic +1. **Use `client.query.builder()` for queries** — it's the primary query pattern; `records.list()` is a shortcut for trivial filter+select only +2. **Schema names are required** - Never use display names +3. **Custom tables need prefixes** - Include customization prefix (e.g., "new_") +4. **Filter is case-sensitive** - Use lowercase logical names +5. **Bulk operations are encouraged** - Pass lists for optimization +6. **No trailing slashes in URLs** - Format: `https://org.crm.dynamics.com` +7. **Structured errors** - Check `is_transient` for retry logic diff --git a/src/PowerPlatform/Dataverse/client.py b/src/PowerPlatform/Dataverse/client.py index 29be479d..c9a1364f 100644 --- a/src/PowerPlatform/Dataverse/client.py +++ b/src/PowerPlatform/Dataverse/client.py @@ -3,9 +3,8 @@ from __future__ import annotations -import warnings from contextlib import contextmanager -from typing import Any, Dict, Iterable, Iterator, List, Optional, Union +from typing import Iterator, Optional import requests @@ -216,581 +215,8 @@ def _check_closed(self) -> None: if self._closed: raise RuntimeError("DataverseClient is closed") - # ---------------- Unified CRUD: create/update/delete ---------------- - def create(self, table_schema_name: str, records: Union[Dict[str, Any], List[Dict[str, Any]]]) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.create` instead. - - Create one or more records by table name. - - :param table_schema_name: Schema name of the table (e.g. ``"account"``, ``"contact"``, or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param records: A single record dictionary or a list of record dictionaries. - Each dictionary should contain column schema names as keys. - :type records: dict or list[dict] - - :return: List of created record GUIDs. Returns a single-element list for a single input. - :rtype: list[str] - - :raises TypeError: If ``records`` is not a dict or list[dict], or if the internal - client returns an unexpected type. - - Example: - Create a single record:: - - client = DataverseClient(base_url, credential) - ids = client.create("account", {"name": "Contoso"}) - print(f"Created: {ids[0]}") - - Create multiple records:: - - records = [ - {"name": "Contoso"}, - {"name": "Fabrikam"} - ] - ids = client.create("account", records) - print(f"Created {len(ids)} accounts") - """ - warnings.warn( - "client.create() is deprecated. Use client.records.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - # Old API always returned list[str]; new returns str for single - if isinstance(records, dict): - return [self.records.create(table_schema_name, records)] - return self.records.create(table_schema_name, records) - - def update( - self, table_schema_name: str, ids: Union[str, List[str]], changes: Union[Dict[str, Any], List[Dict[str, Any]]] - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.update` instead. - - Update one or more records. - - This method supports three usage patterns: - - 1. Single record update: ``update("account", "guid", {"name": "New Name"})`` - 2. Broadcast update: ``update("account", [id1, id2], {"status": 1})`` - applies same changes to all IDs - 3. Paired updates: ``update("account", [id1, id2], [changes1, changes2])`` - one-to-one mapping - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to update. - :type ids: str or list[str] - :param changes: Dictionary of changes for single/broadcast mode, or list of dictionaries - for paired mode. When ``ids`` is a list and ``changes`` is a single dict, - the same changes are broadcast to all records. When both are lists, they must - have equal length for one-to-one mapping. - :type changes: dict or list[dict] - - :raises TypeError: If ``ids`` is not str or list[str], or if ``changes`` type doesn't match usage pattern. - - .. note:: - Single updates discard the response representation for better performance. For broadcast or paired updates, the method delegates to the internal client's batch update logic. - - Example: - Single record update:: - - client.update("account", account_id, {"telephone1": "555-0100"}) - - Broadcast same changes to multiple records:: - - client.update("account", [id1, id2, id3], {"statecode": 1}) - - Update multiple records with different values:: - - ids = [id1, id2] - changes = [ - {"name": "Updated Name 1"}, - {"name": "Updated Name 2"} - ] - client.update("account", ids, changes) - """ - warnings.warn( - "client.update() is deprecated. Use client.records.update() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.records.update(table_schema_name, ids, changes) - - def delete( - self, - table_schema_name: str, - ids: Union[str, List[str]], - use_bulk_delete: bool = True, - ) -> Optional[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.delete` instead. - - Delete one or more records by GUID. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param ids: Single GUID string or list of GUID strings to delete. - :type ids: str or list[str] - :param use_bulk_delete: When ``True`` (default) and ``ids`` is a list, execute the BulkDelete action and - return its async job identifier. When ``False`` each record is deleted sequentially. - :type use_bulk_delete: :class:`bool` - - :raises TypeError: If ``ids`` is not str or list[str]. - :raises HttpError: If the underlying Web API delete request fails. - - :return: BulkDelete job ID when deleting multiple records via BulkDelete; otherwise ``None``. - :rtype: :class:`str` or None - - Example: - Delete a single record:: - - client.delete("account", account_id) - - Delete multiple records:: - - job_id = client.delete("account", [id1, id2, id3]) - """ - warnings.warn( - "client.delete() is deprecated. Use client.records.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.records.delete(table_schema_name, ids, use_bulk_delete=use_bulk_delete) - - def get( - self, - table_schema_name: str, - record_id: Optional[str] = None, - select: Optional[List[str]] = None, - filter: Optional[str] = None, - orderby: Optional[List[str]] = None, - top: Optional[int] = None, - expand: Optional[List[str]] = None, - page_size: Optional[int] = None, - ) -> Union[Dict[str, Any], Iterable[List[Dict[str, Any]]]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.get` instead. - - - **Single record by ID** -- ``client.records.get(table, record_id)`` - - **Query / filter multiple records** -- ``client.records.get(table, filter=..., select=...)`` - - Fetch a single record by ID or query multiple records. - - When ``record_id`` is provided, returns a single record dictionary. - When ``record_id`` is None, returns a generator yielding batches of records. - - :param table_schema_name: Schema name of the table (e.g. ``"account"`` or ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param record_id: Optional GUID to fetch a specific record. If None, queries multiple records. - :type record_id: :class:`str` or None - :param select: Optional list of attribute logical names to retrieve. Column names are case-insensitive and automatically lowercased (e.g. ``["new_Title", "new_Amount"]`` becomes ``"new_title,new_amount"``). - :type select: list[str] or None - :param filter: Optional OData filter string, e.g. ``"name eq 'Contoso'"`` or ``"new_quantity gt 5"``. Column names in filter expressions must use exact lowercase logical names (e.g. ``"new_quantity"``, not ``"new_Quantity"``). The filter string is passed directly to the Dataverse Web API without transformation. - :type filter: :class:`str` or None - :param orderby: Optional list of attributes to sort by, e.g. ``["name asc", "createdon desc"]``. Column names are automatically lowercased. - :type orderby: list[str] or None - :param top: Optional maximum number of records to return. - :type top: :class:`int` or None - :param expand: Optional list of navigation properties to expand, e.g. ``["primarycontactid"]``. Navigation property names are case-sensitive and must match the server-defined names exactly. These are NOT automatically transformed. Consult entity metadata for correct casing. - :type expand: list[str] or None - :param page_size: Optional number of records per page for pagination. - :type page_size: :class:`int` or None - - :return: Single record dict if ``record_id`` is provided, otherwise a generator - yielding lists of record dictionaries (one list per page). - :rtype: dict or collections.abc.Iterable[list[dict]] - - :raises TypeError: If ``record_id`` is provided but not a string. - - Example: - Fetch a single record:: - - record = client.get("account", record_id=account_id, select=["name", "telephone1"]) - print(record["name"]) - - Query multiple records with filtering (note: exact logical names in filter):: - - for batch in client.get( - "account", - filter="statecode eq 0 and name eq 'Contoso'", # Must use exact logical names (lower-case) - select=["name", "telephone1"] - ): - for account in batch: - print(account["name"]) - - Query with navigation property expansion (note: case-sensitive property name):: - - for batch in client.get( - "account", - select=["name"], - expand=["primarycontactid"], # Case-sensitive! Check metadata for exact name - filter="statecode eq 0" - ): - for account in batch: - print(f"{account['name']} - Contact: {account.get('primarycontactid', {}).get('fullname')}") - - Query with sorting and pagination:: - - for batch in client.get( - "account", - orderby=["createdon desc"], - top=100, - page_size=50 - ): - print(f"Batch size: {len(batch)}") - """ - warnings.warn( - "client.get() is deprecated. Use client.records.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - if record_id is not None: - return self.records.get(table_schema_name, record_id, select=select) - else: - return self.records.get( - table_schema_name, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - ) - - # SQL via Web API sql parameter - def query_sql(self, sql: str) -> List[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.query.QueryOperations.sql` instead. - - Execute a read-only SQL query using the Dataverse Web API ``?sql`` capability. - - The SQL query must follow the supported subset: a single SELECT statement with - optional WHERE, TOP (integer literal), ORDER BY (column names only), and a simple - table alias after FROM. - - :param sql: Supported SQL SELECT statement. - :type sql: :class:`str` - - :return: List of result row dictionaries. Returns an empty list if no rows match. - :rtype: list[dict] - - :raises ~PowerPlatform.Dataverse.core.errors.SQLParseError: If the SQL query uses unsupported syntax. - :raises ~PowerPlatform.Dataverse.core.errors.HttpError: If the Web API returns an error. - - .. note:: - The SQL support is limited to read-only queries. Complex joins, subqueries, and certain SQL functions may not be supported. Consult the Dataverse documentation for the current feature set. - - Example: - Basic SQL query:: - - sql = "SELECT TOP 10 accountid, name FROM account WHERE name LIKE 'C%' ORDER BY name" - results = client.query_sql(sql) - for row in results: - print(row["name"]) - - Query with alias:: + # ---------------- Cache utilities ---------------- - sql = "SELECT a.name, a.telephone1 FROM account AS a WHERE a.statecode = 0" - results = client.query_sql(sql) - """ - warnings.warn( - "client.query_sql() is deprecated. Use client.query.sql() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.query.sql(sql) - - # Table metadata helpers - def get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.get` instead. - - Get basic metadata for a table if it exists. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :return: Dictionary containing table metadata with keys ``table_schema_name``, - ``table_logical_name``, ``entity_set_name``, and ``metadata_id``. - Returns None if the table is not found. - :rtype: :class:`dict` or None - - Example: - Retrieve table metadata:: - - info = client.get_table_info("new_MyTestTable") - if info: - print(f"Logical name: {info['table_logical_name']}") - print(f"Entity set: {info['entity_set_name']}") - """ - warnings.warn( - "client.get_table_info() is deprecated. Use client.tables.get() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.get(table_schema_name) - - def create_table( - self, - table_schema_name: str, - columns: Dict[str, Any], - solution_unique_name: Optional[str] = None, - primary_column_schema_name: Optional[str] = None, - ) -> Dict[str, Any]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.create` instead. - - Create a simple custom table with specified columns. - - :param table_schema_name: Schema name of the table with customization prefix value (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Dictionary mapping column names (with customization prefix value) to their types. All custom column names must include the customization prefix value (e.g. ``"new_Title"``). - Supported types: - - - Primitive types: ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"`` - - Enum subclass (IntEnum preferred): Creates a local option set. Optional multilingual - labels can be provided via ``__labels__`` class attribute, defined inside the Enum subclass:: - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - __labels__ = { - 1033: {"Active": "Active", "Inactive": "Inactive"}, - 1036: {"Active": "Actif", "Inactive": "Inactif"} - } - - :type columns: dict[str, typing.Any] - :param solution_unique_name: Optional solution unique name that should own the new table. When omitted the table is created in the default solution. - :type solution_unique_name: :class:`str` or None - :param primary_column_schema_name: Optional primary name column schema name with customization prefix value (e.g. ``"new_MyTestTable"``). If not provided, defaults to ``"{customization prefix value}_Name"``. - :type primary_column_schema_name: :class:`str` or None - - :return: Dictionary containing table metadata including ``table_schema_name``, - ``entity_set_name``, ``table_logical_name``, ``metadata_id``, and ``columns_created``. - :rtype: :class:`dict` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If table creation fails or the schema is invalid. - - Example: - Create a table with simple columns:: - - from enum import IntEnum - - class ItemStatus(IntEnum): - ACTIVE = 1 - INACTIVE = 2 - - columns = { - "new_Title": "string", # Note: includes 'new_' customization prefix value - "new_Quantity": "int", - "new_Price": "decimal", - "new_Available": "bool", - "new_Status": ItemStatus - } - - result = client.create_table("new_MyTestTable", columns) - print(f"Created table: {result['table_schema_name']}") - print(f"Columns: {result['columns_created']}") - - Create a table with a custom primary column name:: - - result = client.create_table( - "new_Product", - {"new_Price": "decimal"}, - primary_column_schema_name="new_ProductName" - ) - """ - warnings.warn( - "client.create_table() is deprecated. Use client.tables.create() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.create( - table_schema_name, - columns, - solution=solution_unique_name, - primary_column=primary_column_schema_name, - ) - - def delete_table(self, table_schema_name: str) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.delete` instead. - - Delete a custom table by name. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"`` or ``"account"``). - :type table_schema_name: :class:`str` - - :raises ~PowerPlatform.Dataverse.core.errors.MetadataError: If the table does not exist or deletion fails. - - .. warning:: - This operation is irreversible and will delete all records in the table along - with the table definition. Use with caution. - - Example: - Delete a custom table:: - - client.delete_table("new_MyTestTable") - """ - warnings.warn( - "client.delete_table() is deprecated. Use client.tables.delete() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.tables.delete(table_schema_name) - - def list_tables(self) -> list[dict[str, Any]]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.list` instead. - - List all non-private tables in the Dataverse environment. - - :return: List of EntityDefinition metadata dictionaries. - :rtype: list[dict] - - Example: - List all non-private tables and print their logical names:: - - tables = client.list_tables() - for table in tables: - print(table["LogicalName"]) - """ - warnings.warn( - "client.list_tables() is deprecated. Use client.tables.list() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.list() - - def create_columns( - self, - table_schema_name: str, - columns: Dict[str, Any], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.add_columns` instead. - - Create one or more columns on an existing table using a schema-style mapping. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Mapping of column schema names (with customization prefix value) to supported types. All custom column names must include the customization prefix value** (e.g. ``"new_Notes"``). Primitive types include - ``"string"`` (alias: ``"text"``), ``"int"`` (alias: ``"integer"``), ``"decimal"`` (alias: ``"money"``), ``"float"`` (alias: ``"double"``), ``"datetime"`` (alias: ``"date"``), ``"bool"`` (alias: ``"boolean"``), and ``"file"``. Enum subclasses (IntEnum preferred) - generate a local option set and can specify localized labels via ``__labels__``. - :type columns: dict[str, typing.Any] - :returns: Schema names for the columns that were created. - :rtype: list[str] - Example: - Create multiple columns on the custom table:: - - created = client.create_columns( - "new_MyTestTable", - { - "new_Scratch": "string", - "new_Flags": "bool", - "new_Document": "file", - }, - ) - print(created) # ['new_Scratch', 'new_Flags', 'new_Document'] - """ - warnings.warn( - "client.create_columns() is deprecated. Use client.tables.add_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.add_columns(table_schema_name, columns) - - def delete_columns( - self, - table_schema_name: str, - columns: Union[str, List[str]], - ) -> List[str]: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.tables.TableOperations.remove_columns` instead. - - Delete one or more columns from a table. - - :param table_schema_name: Schema name of the table (e.g. ``"new_MyTestTable"``). - :type table_schema_name: :class:`str` - :param columns: Column name or list of column names to remove. Must include customization prefix value (e.g. ``"new_TestColumn"``). - :type columns: str or list[str] - :returns: Schema names for the columns that were removed. - :rtype: list[str] - Example: - Remove two custom columns by schema name: - - removed = client.delete_columns( - "new_MyTestTable", - ["new_Scratch", "new_Flags"], - ) - print(removed) # ['new_Scratch', 'new_Flags'] - """ - warnings.warn( - "client.delete_columns() is deprecated. Use client.tables.remove_columns() instead.", - DeprecationWarning, - stacklevel=2, - ) - return self.tables.remove_columns(table_schema_name, columns) - - # File upload - def upload_file( - self, - table_schema_name: str, - record_id: str, - file_name_attribute: str, - path: str, - mode: Optional[str] = None, - mime_type: Optional[str] = None, - if_none_match: bool = True, - ) -> None: - """ - .. note:: - Deprecated. Use :meth:`~PowerPlatform.Dataverse.operations.files.FileOperations.upload` instead. - - Upload a file to a Dataverse file column. - - :param table_schema_name: Schema name of the table. - :type table_schema_name: :class:`str` - :param record_id: GUID of the target record. - :type record_id: :class:`str` - :param file_name_attribute: Schema name of the file column attribute. - :type file_name_attribute: :class:`str` - :param path: Local filesystem path to the file. - :type path: :class:`str` - :param mode: Upload strategy: ``"auto"`` (default), ``"small"``, or ``"chunk"``. - :type mode: :class:`str` or None - :param mime_type: Explicit MIME type to store with the file. - :type mime_type: :class:`str` or None - :param if_none_match: When True (default), only succeed if the column is - currently empty. - :type if_none_match: :class:`bool` - """ - warnings.warn( - "client.upload_file() is deprecated. Use client.files.upload() instead.", - DeprecationWarning, - stacklevel=2, - ) - self.files.upload( - table_schema_name, - record_id, - file_name_attribute, - path, - mode=mode, - mime_type=mime_type, - if_none_match=if_none_match, - ) - - # Cache utilities def flush_cache(self, kind) -> int: """ Flush cached client metadata or state. diff --git a/src/PowerPlatform/Dataverse/data/_batch.py b/src/PowerPlatform/Dataverse/data/_batch.py index 2e880c01..b19aac55 100644 --- a/src/PowerPlatform/Dataverse/data/_batch.py +++ b/src/PowerPlatform/Dataverse/data/_batch.py @@ -5,224 +5,50 @@ from __future__ import annotations -import json -import re import uuid -from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union - -from ..core.errors import HttpError, MetadataError, ValidationError -from ..core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND, _http_subcode -from ..models.batch import BatchItemResponse, BatchResult -from ..models.relationship import ( - LookupAttributeMetadata, - OneToManyRelationshipMetadata, - ManyToManyRelationshipMetadata, -) -from ..models.upsert import UpsertItem -from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union + +from ..core.errors import MetadataError, ValidationError +from ..core._error_codes import METADATA_TABLE_NOT_FOUND, METADATA_COLUMN_NOT_FOUND +from ..models.batch import BatchResult from ._raw_request import _RawRequest -from ._odata import _GUID_RE +from ._batch_base import ( + _BatchBase, + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, + _ChangeSetBatchItem, + _MAX_BATCH_SIZE, +) if TYPE_CHECKING: from ._odata import _ODataClient __all__ = [] -_CRLF = "\r\n" -_MAX_BATCH_SIZE = 1000 - - -# --------------------------------------------------------------------------- -# Intent dataclasses — one per supported operation type -# (stored at batch-build time; resolved to _RawRequest at execute() time) -# --------------------------------------------------------------------------- - -# --- Record intent types --- - - -@dataclass -class _RecordCreate: - table: str - data: Union[Dict[str, Any], List[Dict[str, Any]]] - content_id: Optional[int] = None # set only for changeset items - - -@dataclass -class _RecordUpdate: - table: str - ids: Union[str, List[str]] - changes: Union[Dict[str, Any], List[Dict[str, Any]]] - content_id: Optional[int] = None # set only for changeset single-record updates - - -@dataclass -class _RecordDelete: - table: str - ids: Union[str, List[str]] - use_bulk_delete: bool = True - content_id: Optional[int] = None # set only for changeset single-record deletes - - -@dataclass -class _RecordGet: - table: str - record_id: str - select: Optional[List[str]] = None - - -@dataclass -class _RecordUpsert: - table: str - items: List[UpsertItem] # always non-empty; normalised by BatchRecordOperations - - -# --- Table intent types --- - - -@dataclass -class _TableCreate: - table: str - columns: Dict[str, Any] - solution: Optional[str] = None - primary_column: Optional[str] = None - display_name: Optional[str] = None - - -@dataclass -class _TableDelete: - table: str - - -@dataclass -class _TableGet: - table: str - - -@dataclass -class _TableList: - filter: Optional[str] = None - select: Optional[List[str]] = None - - -@dataclass -class _TableAddColumns: - table: str - columns: Dict[str, Any] - - -@dataclass -class _TableRemoveColumns: - table: str - columns: Union[str, List[str]] - - -@dataclass -class _TableCreateOneToMany: - lookup: LookupAttributeMetadata - relationship: OneToManyRelationshipMetadata - solution: Optional[str] = None - - -@dataclass -class _TableCreateManyToMany: - relationship: ManyToManyRelationshipMetadata - solution: Optional[str] = None - - -@dataclass -class _TableDeleteRelationship: - relationship_id: str - - -@dataclass -class _TableGetRelationship: - schema_name: str - - -@dataclass -class _TableCreateLookupField: - referencing_table: str - lookup_field_name: str - referenced_table: str - display_name: Optional[str] = None - description: Optional[str] = None - required: bool = False - cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK - solution: Optional[str] = None - language_code: int = 1033 - - -# --- Query intent types --- - - -@dataclass -class _QuerySql: - sql: str - - -# --------------------------------------------------------------------------- -# Changeset container -# --------------------------------------------------------------------------- - - -@dataclass -class _ChangeSet: - """Ordered group of single-record write operations that execute atomically. - - Content-IDs are allocated from ``_counter``, a single-element ``List[int]`` - that is shared across all changesets in the same batch. Passing the same - list object to every ``_ChangeSet`` created by a :class:`BatchRequest` - ensures Content-ID values are unique within the entire batch request, not - just within an individual changeset, as required by the OData spec. - - When constructed in isolation (e.g. in unit tests), ``_counter`` defaults - to a fresh ``[1]`` so the class remains self-contained. - """ - - operations: List[Union[_RecordCreate, _RecordUpdate, _RecordDelete]] = field(default_factory=list) - _counter: List[int] = field(default_factory=lambda: [1], repr=False) - - def add_create(self, table: str, data: Dict[str, Any]) -> str: - """Add a single-record create; return its content-ID reference string.""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordCreate(table=table, data=data, content_id=cid)) - return f"${cid}" - - def add_update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: - """Add a single-record update (record_id may be a '$n' reference).""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordUpdate(table=table, ids=record_id, changes=changes, content_id=cid)) - - def add_delete(self, table: str, record_id: str) -> None: - """Add a single-record delete (record_id may be a '$n' reference).""" - cid = self._counter[0] - self._counter[0] += 1 - self.operations.append(_RecordDelete(table=table, ids=record_id, content_id=cid)) - - -# --------------------------------------------------------------------------- -# Changeset batch item -# (_RawRequest is imported from ._raw_request — defined there so _odata.py -# can also import it without a circular dependency) -# --------------------------------------------------------------------------- - - -@dataclass -class _ChangeSetBatchItem: - """A resolved changeset — serialised as a nested multipart/mixed part.""" - - requests: List[_RawRequest] - # --------------------------------------------------------------------------- # Batch client: resolves intents → raw requests → multipart body → HTTP → result # --------------------------------------------------------------------------- -class _BatchClient: +class _BatchClient(_BatchBase): """ Serialises a list of intent objects into an OData ``$batch`` multipart/mixed request, dispatches it, and parses the response. @@ -230,9 +56,6 @@ class _BatchClient: :param od: The active OData client (provides helpers and HTTP transport). """ - def __init__(self, od: "_ODataClient") -> None: - self._od = od - # ------------------------------------------------------------------ # Public entry point # ------------------------------------------------------------------ @@ -312,6 +135,8 @@ def _resolve_item(self, item: Any) -> List[_RawRequest]: return self._resolve_record_delete(item) if isinstance(item, _RecordGet): return self._resolve_record_get(item) + if isinstance(item, _RecordList): + return self._resolve_record_list(item) if isinstance(item, _RecordUpsert): return self._resolve_record_upsert(item) if isinstance(item, _TableCreate): @@ -382,7 +207,30 @@ def _resolve_record_delete(self, op: _RecordDelete) -> List[_RawRequest]: return [self._od._build_delete(op.table, rid) for rid in ids] def _resolve_record_get(self, op: _RecordGet) -> List[_RawRequest]: - return [self._od._build_get(op.table, op.record_id, select=op.select)] + return [ + self._od._build_get( + op.table, + op.record_id, + select=op.select, + expand=op.expand, + include_annotations=op.include_annotations, + ) + ] + + def _resolve_record_list(self, op: _RecordList) -> List[_RawRequest]: + return [ + self._od._build_list( + op.table, + select=op.select, + filter=op.filter, + orderby=op.orderby, + top=op.top, + expand=op.expand, + page_size=op.page_size, + count=op.count, + include_annotations=op.include_annotations, + ) + ] def _resolve_record_upsert(self, op: _RecordUpsert) -> List[_RawRequest]: entity_set = self._od._entity_set_from_schema_name(op.table) @@ -409,19 +257,10 @@ def _require_entity_metadata(self, table: str) -> str: ) return ent["MetadataId"] - def _resolve_table_create(self, op: _TableCreate) -> List[_RawRequest]: - return [self._od._build_create_entity(op.table, op.columns, op.solution, op.primary_column, op.display_name)] - def _resolve_table_delete(self, op: _TableDelete) -> List[_RawRequest]: metadata_id = self._require_entity_metadata(op.table) return [self._od._build_delete_entity(metadata_id)] - def _resolve_table_get(self, op: _TableGet) -> List[_RawRequest]: - return [self._od._build_get_entity(op.table)] - - def _resolve_table_list(self, op: _TableList) -> List[_RawRequest]: - return [self._od._build_list_entities(filter=op.filter, select=op.select)] - def _resolve_table_add_columns(self, op: _TableAddColumns) -> List[_RawRequest]: metadata_id = self._require_entity_metadata(op.table) return [self._od._build_create_column(metadata_id, col_name, dtype) for col_name, dtype in op.columns.items()] @@ -442,255 +281,9 @@ def _resolve_table_remove_columns(self, op: _TableRemoveColumns) -> List[_RawReq requests.append(self._od._build_delete_column(metadata_id, attr_meta["MetadataId"])) return requests - def _resolve_table_create_one_to_many(self, op: _TableCreateOneToMany) -> List[_RawRequest]: - body = op.relationship.to_dict() - body["Lookup"] = op.lookup.to_dict() - return [self._od._build_create_relationship(body, solution=op.solution)] - - def _resolve_table_create_many_to_many(self, op: _TableCreateManyToMany) -> List[_RawRequest]: - return [self._od._build_create_relationship(op.relationship.to_dict(), solution=op.solution)] - - def _resolve_table_delete_relationship(self, op: _TableDeleteRelationship) -> List[_RawRequest]: - return [self._od._build_delete_relationship(op.relationship_id)] - - def _resolve_table_get_relationship(self, op: _TableGetRelationship) -> List[_RawRequest]: - return [self._od._build_get_relationship(op.schema_name)] - - def _resolve_table_create_lookup_field(self, op: _TableCreateLookupField) -> List[_RawRequest]: - lookup, relationship = self._od._build_lookup_field_models( - referencing_table=op.referencing_table, - lookup_field_name=op.lookup_field_name, - referenced_table=op.referenced_table, - display_name=op.display_name, - description=op.description, - required=op.required, - cascade_delete=op.cascade_delete, - language_code=op.language_code, - ) - body = relationship.to_dict() - body["Lookup"] = lookup.to_dict() - return [self._od._build_create_relationship(body, solution=op.solution)] - # ------------------------------------------------------------------ # Query resolvers — delegate to _ODataClient._build_* methods # ------------------------------------------------------------------ def _resolve_query_sql(self, op: _QuerySql) -> List[_RawRequest]: return [self._od._build_sql(op.sql)] - - # ------------------------------------------------------------------ - # Multipart serialisation - # ------------------------------------------------------------------ - - def _build_batch_body( - self, - resolved: List[Union[_RawRequest, _ChangeSetBatchItem]], - batch_boundary: str, - ) -> str: - parts: List[str] = [] - for item in resolved: - if isinstance(item, _ChangeSetBatchItem): - parts.append(self._serialize_changeset_item(item, batch_boundary)) - else: - parts.append(self._serialize_raw_request(item, batch_boundary)) - return "".join(parts) + f"--{batch_boundary}--{_CRLF}" - - def _serialize_raw_request(self, req: _RawRequest, boundary: str) -> str: - """Serialise a single operation as a multipart/mixed part with CRLF line endings.""" - part_header_lines = [ - f"--{boundary}", - "Content-Type: application/http", - "Content-Transfer-Encoding: binary", - ] - if req.content_id is not None: - part_header_lines.append(f"Content-ID: {req.content_id}") - - inner_lines = [f"{req.method} {req.url} HTTP/1.1"] - if req.body is not None: - inner_lines.append("Content-Type: application/json; type=entry") - if req.headers: - for k, v in req.headers.items(): - inner_lines.append(f"{k}: {v}") - inner_lines.append("") # blank line — end of inner headers - if req.body is not None: - inner_lines.append(req.body) - - part_header_str = _CRLF.join(part_header_lines) + _CRLF - inner_str = _CRLF.join(inner_lines) - return part_header_str + _CRLF + inner_str + _CRLF - - def _serialize_changeset_item(self, cs: _ChangeSetBatchItem, batch_boundary: str) -> str: - cs_boundary = f"changeset_{uuid.uuid4()}" - cs_parts = [self._serialize_raw_request(r, cs_boundary) for r in cs.requests] - cs_parts.append(f"--{cs_boundary}--{_CRLF}") - cs_body = "".join(cs_parts) - - outer = ( - f"--{batch_boundary}{_CRLF}" f'Content-Type: multipart/mixed; boundary="{cs_boundary}"{_CRLF}' f"{_CRLF}" - ) - return outer + cs_body + _CRLF - - # ------------------------------------------------------------------ - # Response parsing (multipart/mixed) - # ------------------------------------------------------------------ - - def _parse_batch_response(self, response: Any) -> BatchResult: - content_type = response.headers.get("Content-Type", "") - boundary = _extract_boundary(content_type) - if not boundary: - # Non-multipart response: the batch request itself was rejected by Dataverse - # (common for top-level 4xx, e.g. malformed body, missing OData headers). - # Returning an empty BatchResult() here would silently hide the error and - # make has_errors=False, which is actively misleading. Raise instead. - _raise_top_level_batch_error(response) - return BatchResult() # unreachable; satisfies type checkers - parts = _split_multipart(response.text or "", boundary) - responses: List[BatchItemResponse] = [] - for part_headers, part_body in parts: - part_ct = part_headers.get("content-type", "") - if "multipart/mixed" in part_ct: - inner_boundary = _extract_boundary(part_ct) - if inner_boundary: - for ih, ib in _split_multipart(part_body, inner_boundary): - item = _parse_http_response_part(ib, ih.get("content-id")) - if item is not None: - responses.append(item) - else: - item = _parse_http_response_part(part_body, content_id=part_headers.get("content-id")) - if item is not None: - responses.append(item) - return BatchResult(responses=responses) - - -# --------------------------------------------------------------------------- -# Multipart parsing helpers -# --------------------------------------------------------------------------- - - -def _raise_top_level_batch_error(response: Any) -> None: - """Parse a non-multipart batch response and raise HttpError with the service message. - - Dataverse returns ``application/json`` with an ``{"error": {...}}`` payload when - it rejects the batch request at the HTTP level (e.g. malformed multipart body, - missing OData headers). This helper surfaces that detail instead of silently - returning an empty ``BatchResult``. - """ - status_code: int = getattr(response, "status_code", 0) - service_error_code: Optional[str] = None - try: - payload = response.json() - error = payload.get("error", {}) - service_error_code = error.get("code") or None - message: str = error.get("message") or response.text or "Unexpected non-multipart response from $batch" - except Exception: - message = (getattr(response, "text", None) or "") or "Unexpected non-multipart response from $batch" - raise HttpError( - message=f"Batch request rejected by Dataverse: {message}", - status_code=status_code, - subcode=_http_subcode(status_code) if status_code else None, - service_error_code=service_error_code, - ) - - -_BOUNDARY_RE = re.compile(r'boundary="?([^";,\s]+)"?', re.IGNORECASE) - - -def _extract_boundary(content_type: str) -> Optional[str]: - m = _BOUNDARY_RE.search(content_type) - return m.group(1) if m else None - - -def _split_multipart(body: str, boundary: str) -> List[Tuple[Dict[str, str], str]]: - delimiter = f"--{boundary}" - parts: List[Tuple[Dict[str, str], str]] = [] - lines = body.replace("\r\n", "\n").split("\n") - current: List[str] = [] - in_part = False - for line in lines: - stripped = line.rstrip("\r") - if stripped == delimiter: - if in_part and current: - parts.append(_parse_mime_part("\n".join(current))) - current = [] - in_part = True - elif stripped == f"{delimiter}--": - if in_part and current: - parts.append(_parse_mime_part("\n".join(current))) - break - elif in_part: - current.append(line) - return parts - - -def _parse_mime_part(raw: str) -> Tuple[Dict[str, str], str]: - if "\n\n" in raw: - header_block, body = raw.split("\n\n", 1) - else: - header_block, body = raw, "" - headers: Dict[str, str] = {} - for line in header_block.splitlines(): - if ":" in line: - name, _, value = line.partition(":") - headers[name.strip().lower()] = value.strip() - return headers, body.strip() - - -def _parse_http_response_part(text: str, content_id: Optional[str]) -> Optional[BatchItemResponse]: - lines = text.replace("\r\n", "\n").splitlines() - if not lines: - return None - status_line = "" - idx = 0 - for i, line in enumerate(lines): - if line.startswith("HTTP/"): - status_line = line - idx = i + 1 - break - if not status_line: - return None - parts = status_line.split(" ", 2) - if len(parts) < 2: - return None - try: - status_code = int(parts[1]) - except ValueError: - return None - resp_headers: Dict[str, str] = {} - body_start = idx - for i in range(idx, len(lines)): - if lines[i] == "": - body_start = i + 1 - break - if ":" in lines[i]: - name, _, value = lines[i].partition(":") - resp_headers[name.strip().lower()] = value.strip() - entity_id: Optional[str] = None - odata_id = resp_headers.get("odata-entityid", "") - if odata_id: - m = _GUID_RE.search(odata_id) - if m: - entity_id = m.group(0) - body_text = "\n".join(lines[body_start:]).strip() - data: Optional[Dict[str, Any]] = None - error_message: Optional[str] = None - error_code: Optional[str] = None - if body_text: - try: - parsed = json.loads(body_text) - if isinstance(parsed, dict): - err = parsed.get("error") - if isinstance(err, dict): - error_message = err.get("message") - error_code = err.get("code") - else: - data = parsed - except (json.JSONDecodeError, ValueError): - pass - return BatchItemResponse( - status_code=status_code, - content_id=content_id, - entity_id=entity_id, - data=data, - error_message=error_message, - error_code=error_code, - ) diff --git a/src/PowerPlatform/Dataverse/data/_batch_base.py b/src/PowerPlatform/Dataverse/data/_batch_base.py new file mode 100644 index 00000000..46b247de --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_batch_base.py @@ -0,0 +1,513 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared intent types, multipart helpers, and pure-logic base for the Dataverse batch client. + +Contains no I/O. Subclasses add the HTTP transport layer (sync or async). +""" + +from __future__ import annotations + +import json +import re +import uuid +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union + +from ..core.errors import HttpError, ValidationError +from ..core._error_codes import _http_subcode +from ..models.batch import BatchItemResponse, BatchResult +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from ..models.upsert import UpsertItem +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from ._raw_request import _RawRequest +from ._odata_base import _GUID_RE + +if TYPE_CHECKING: + from ._odata_base import _ODataBase + +__all__ = [] + +_CRLF = "\r\n" +_MAX_BATCH_SIZE = 1000 + + +# --------------------------------------------------------------------------- +# Intent dataclasses — one per supported operation type +# (stored at batch-build time; resolved to _RawRequest at execute() time) +# --------------------------------------------------------------------------- + +# --- Record intent types --- + + +@dataclass +class _RecordCreate: + table: str + data: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset items + + +@dataclass +class _RecordUpdate: + table: str + ids: Union[str, List[str]] + changes: Union[Dict[str, Any], List[Dict[str, Any]]] + content_id: Optional[int] = None # set only for changeset single-record updates + + +@dataclass +class _RecordDelete: + table: str + ids: Union[str, List[str]] + use_bulk_delete: bool = True + content_id: Optional[int] = None # set only for changeset single-record deletes + + +@dataclass +class _RecordGet: + table: str + record_id: str + select: Optional[List[str]] = None + expand: Optional[List[str]] = None + include_annotations: Optional[str] = None + + +@dataclass +class _RecordList: + table: str + select: Optional[List[str]] = None + filter: Optional[str] = None + orderby: Optional[List[str]] = None + top: Optional[int] = None + expand: Optional[List[str]] = None + page_size: Optional[int] = None + count: bool = False + include_annotations: Optional[str] = None + + +@dataclass +class _RecordUpsert: + table: str + items: List[UpsertItem] # always non-empty; normalised by BatchRecordOperations + + +# --- Table intent types --- + + +@dataclass +class _TableCreate: + table: str + columns: Dict[str, Any] + solution: Optional[str] = None + primary_column: Optional[str] = None + display_name: Optional[str] = None + + +@dataclass +class _TableDelete: + table: str + + +@dataclass +class _TableGet: + table: str + + +@dataclass +class _TableList: + filter: Optional[str] = None + select: Optional[List[str]] = None + + +@dataclass +class _TableAddColumns: + table: str + columns: Dict[str, Any] + + +@dataclass +class _TableRemoveColumns: + table: str + columns: Union[str, List[str]] + + +@dataclass +class _TableCreateOneToMany: + lookup: LookupAttributeMetadata + relationship: OneToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableCreateManyToMany: + relationship: ManyToManyRelationshipMetadata + solution: Optional[str] = None + + +@dataclass +class _TableDeleteRelationship: + relationship_id: str + + +@dataclass +class _TableGetRelationship: + schema_name: str + + +@dataclass +class _TableCreateLookupField: + referencing_table: str + lookup_field_name: str + referenced_table: str + display_name: Optional[str] = None + description: Optional[str] = None + required: bool = False + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK + solution: Optional[str] = None + language_code: int = 1033 + + +# --- Query intent types --- + + +@dataclass +class _QuerySql: + sql: str + + +# --------------------------------------------------------------------------- +# Changeset container +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSet: + """Ordered group of single-record write operations that execute atomically. + + Content-IDs are allocated from ``_counter``, a single-element ``List[int]`` + that is shared across all changesets in the same batch. Passing the same + list object to every ``_ChangeSet`` created by a :class:`BatchRequest` + ensures Content-ID values are unique within the entire batch request, not + just within an individual changeset, as required by the OData spec. + + When constructed in isolation (e.g. in unit tests), ``_counter`` defaults + to a fresh ``[1]`` so the class remains self-contained. + """ + + operations: List[Union[_RecordCreate, _RecordUpdate, _RecordDelete]] = field(default_factory=list) + _counter: List[int] = field(default_factory=lambda: [1], repr=False) + + def add_create(self, table: str, data: Dict[str, Any]) -> str: + """Add a single-record create; return its content-ID reference string.""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordCreate(table=table, data=data, content_id=cid)) + return f"${cid}" + + def add_update(self, table: str, record_id: str, changes: Dict[str, Any]) -> None: + """Add a single-record update (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordUpdate(table=table, ids=record_id, changes=changes, content_id=cid)) + + def add_delete(self, table: str, record_id: str) -> None: + """Add a single-record delete (record_id may be a '$n' reference).""" + cid = self._counter[0] + self._counter[0] += 1 + self.operations.append(_RecordDelete(table=table, ids=record_id, content_id=cid)) + + +# --------------------------------------------------------------------------- +# Changeset batch item +# (_RawRequest is imported from ._raw_request — defined there so _odata.py +# can also import it without a circular dependency) +# --------------------------------------------------------------------------- + + +@dataclass +class _ChangeSetBatchItem: + """A resolved changeset — serialised as a nested multipart/mixed part.""" + + requests: List[_RawRequest] + + +# --------------------------------------------------------------------------- +# Batch base: pure serialisation and pure table resolvers +# --------------------------------------------------------------------------- + + +class _BatchBase: + """Pure-logic base for the Dataverse batch client. + + Provides multipart serialisation, response parsing, and the subset of + intent resolvers that require no I/O. Subclasses must supply ``execute`` + and the I/O-dependent resolvers. + + :param od: The active OData client (provides helpers and HTTP transport). + """ + + def __init__(self, od: "_ODataBase") -> None: + self._od = od + + # ------------------------------------------------------------------ + # Pure table resolvers — delegate to _ODataBase._build_* methods + # ------------------------------------------------------------------ + + def _resolve_table_create(self, op: _TableCreate) -> List[_RawRequest]: + return [self._od._build_create_entity(op.table, op.columns, op.solution, op.primary_column, op.display_name)] + + def _resolve_table_get(self, op: _TableGet) -> List[_RawRequest]: + return [self._od._build_get_entity(op.table)] + + def _resolve_table_list(self, op: _TableList) -> List[_RawRequest]: + return [self._od._build_list_entities(filter=op.filter, select=op.select)] + + def _resolve_table_create_one_to_many(self, op: _TableCreateOneToMany) -> List[_RawRequest]: + body = op.relationship.to_dict() + body["Lookup"] = op.lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + def _resolve_table_create_many_to_many(self, op: _TableCreateManyToMany) -> List[_RawRequest]: + return [self._od._build_create_relationship(op.relationship.to_dict(), solution=op.solution)] + + def _resolve_table_delete_relationship(self, op: _TableDeleteRelationship) -> List[_RawRequest]: + return [self._od._build_delete_relationship(op.relationship_id)] + + def _resolve_table_get_relationship(self, op: _TableGetRelationship) -> List[_RawRequest]: + return [self._od._build_get_relationship(op.schema_name)] + + def _resolve_table_create_lookup_field(self, op: _TableCreateLookupField) -> List[_RawRequest]: + lookup, relationship = self._od._build_lookup_field_models( + referencing_table=op.referencing_table, + lookup_field_name=op.lookup_field_name, + referenced_table=op.referenced_table, + display_name=op.display_name, + description=op.description, + required=op.required, + cascade_delete=op.cascade_delete, + language_code=op.language_code, + ) + body = relationship.to_dict() + body["Lookup"] = lookup.to_dict() + return [self._od._build_create_relationship(body, solution=op.solution)] + + # ------------------------------------------------------------------ + # Multipart serialisation + # ------------------------------------------------------------------ + + def _build_batch_body( + self, + resolved: List[Union[_RawRequest, _ChangeSetBatchItem]], + batch_boundary: str, + ) -> str: + parts: List[str] = [] + for item in resolved: + if isinstance(item, _ChangeSetBatchItem): + parts.append(self._serialize_changeset_item(item, batch_boundary)) + else: + parts.append(self._serialize_raw_request(item, batch_boundary)) + return "".join(parts) + f"--{batch_boundary}--{_CRLF}" + + def _serialize_raw_request(self, req: _RawRequest, boundary: str) -> str: + """Serialise a single operation as a multipart/mixed part with CRLF line endings.""" + part_header_lines = [ + f"--{boundary}", + "Content-Type: application/http", + "Content-Transfer-Encoding: binary", + ] + if req.content_id is not None: + part_header_lines.append(f"Content-ID: {req.content_id}") + + inner_lines = [f"{req.method} {req.url} HTTP/1.1"] + if req.body is not None: + inner_lines.append("Content-Type: application/json; type=entry") + if req.headers: + for k, v in req.headers.items(): + inner_lines.append(f"{k}: {v}") + inner_lines.append("") # blank line — end of inner headers + if req.body is not None: + inner_lines.append(req.body) + + part_header_str = _CRLF.join(part_header_lines) + _CRLF + inner_str = _CRLF.join(inner_lines) + return part_header_str + _CRLF + inner_str + _CRLF + + def _serialize_changeset_item(self, cs: _ChangeSetBatchItem, batch_boundary: str) -> str: + cs_boundary = f"changeset_{uuid.uuid4()}" + cs_parts = [self._serialize_raw_request(r, cs_boundary) for r in cs.requests] + cs_parts.append(f"--{cs_boundary}--{_CRLF}") + cs_body = "".join(cs_parts) + + outer = ( + f"--{batch_boundary}{_CRLF}" f'Content-Type: multipart/mixed; boundary="{cs_boundary}"{_CRLF}' f"{_CRLF}" + ) + return outer + cs_body + _CRLF + + # ------------------------------------------------------------------ + # Response parsing (multipart/mixed) + # ------------------------------------------------------------------ + + def _parse_batch_response(self, response: Any) -> BatchResult: + content_type = response.headers.get("Content-Type", "") + boundary = _extract_boundary(content_type) + if not boundary: + # Non-multipart response: the batch request itself was rejected by Dataverse + # (common for top-level 4xx, e.g. malformed body, missing OData headers). + # Returning an empty BatchResult() here would silently hide the error and + # make has_errors=False, which is actively misleading. Raise instead. + _raise_top_level_batch_error(response) + return BatchResult() # unreachable; satisfies type checkers + parts = _split_multipart(response.text or "", boundary) + responses: List[BatchItemResponse] = [] + for part_headers, part_body in parts: + part_ct = part_headers.get("content-type", "") + if "multipart/mixed" in part_ct: + inner_boundary = _extract_boundary(part_ct) + if inner_boundary: + for ih, ib in _split_multipart(part_body, inner_boundary): + item = _parse_http_response_part(ib, ih.get("content-id")) + if item is not None: + responses.append(item) + else: + item = _parse_http_response_part(part_body, content_id=part_headers.get("content-id")) + if item is not None: + responses.append(item) + return BatchResult(responses=responses) + + +# --------------------------------------------------------------------------- +# Multipart parsing helpers +# --------------------------------------------------------------------------- + + +def _raise_top_level_batch_error(response: Any) -> None: + """Parse a non-multipart batch response and raise HttpError with the service message. + + Dataverse returns ``application/json`` with an ``{"error": {...}}`` payload when + it rejects the batch request at the HTTP level (e.g. malformed multipart body, + missing OData headers). This helper surfaces that detail instead of silently + returning an empty ``BatchResult``. + """ + status_code: int = getattr(response, "status_code", 0) + service_error_code: Optional[str] = None + try: + payload = response.json() + error = payload.get("error", {}) + service_error_code = error.get("code") or None + message: str = error.get("message") or response.text or "Unexpected non-multipart response from $batch" + except Exception: + message = (getattr(response, "text", None) or "") or "Unexpected non-multipart response from $batch" + raise HttpError( + message=f"Batch request rejected by Dataverse: {message}", + status_code=status_code, + subcode=_http_subcode(status_code) if status_code else None, + service_error_code=service_error_code, + ) + + +_BOUNDARY_RE = re.compile(r'boundary="?([^";,\s]+)"?', re.IGNORECASE) + + +def _extract_boundary(content_type: str) -> Optional[str]: + m = _BOUNDARY_RE.search(content_type) + return m.group(1) if m else None + + +def _split_multipart(body: str, boundary: str) -> List[Tuple[Dict[str, str], str]]: + delimiter = f"--{boundary}" + parts: List[Tuple[Dict[str, str], str]] = [] + lines = body.replace("\r\n", "\n").split("\n") + current: List[str] = [] + in_part = False + for line in lines: + stripped = line.rstrip("\r") + if stripped == delimiter: + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + current = [] + in_part = True + elif stripped == f"{delimiter}--": + if in_part and current: + parts.append(_parse_mime_part("\n".join(current))) + break + elif in_part: + current.append(line) + return parts + + +def _parse_mime_part(raw: str) -> Tuple[Dict[str, str], str]: + if "\n\n" in raw: + header_block, body = raw.split("\n\n", 1) + else: + header_block, body = raw, "" + headers: Dict[str, str] = {} + for line in header_block.splitlines(): + if ":" in line: + name, _, value = line.partition(":") + headers[name.strip().lower()] = value.strip() + return headers, body.strip() + + +def _parse_http_response_part(text: str, content_id: Optional[str]) -> Optional[BatchItemResponse]: + lines = text.replace("\r\n", "\n").splitlines() + if not lines: + return None + status_line = "" + idx = 0 + for i, line in enumerate(lines): + if line.startswith("HTTP/"): + status_line = line + idx = i + 1 + break + if not status_line: + return None + parts = status_line.split(" ", 2) + if len(parts) < 2: + return None + try: + status_code = int(parts[1]) + except ValueError: + return None + resp_headers: Dict[str, str] = {} + body_start = idx + for i in range(idx, len(lines)): + if lines[i] == "": + body_start = i + 1 + break + if ":" in lines[i]: + name, _, value = lines[i].partition(":") + resp_headers[name.strip().lower()] = value.strip() + entity_id: Optional[str] = None + odata_id = resp_headers.get("odata-entityid", "") + if odata_id: + m = _GUID_RE.search(odata_id) + if m: + entity_id = m.group(0) + body_text = "\n".join(lines[body_start:]).strip() + data: Optional[Dict[str, Any]] = None + error_message: Optional[str] = None + error_code: Optional[str] = None + if body_text: + try: + parsed = json.loads(body_text) + if isinstance(parsed, dict): + err = parsed.get("error") + if isinstance(err, dict): + error_message = err.get("message") + error_code = err.get("code") + else: + data = parsed + except (json.JSONDecodeError, ValueError): + pass + return BatchItemResponse( + status_code=status_code, + content_id=content_id, + entity_id=entity_id, + data=data, + error_message=error_message, + error_code=error_code, + ) diff --git a/src/PowerPlatform/Dataverse/data/_odata.py b/src/PowerPlatform/Dataverse/data/_odata.py index b6cdf29a..2264ccf9 100644 --- a/src/PowerPlatform/Dataverse/data/_odata.py +++ b/src/PowerPlatform/Dataverse/data/_odata.py @@ -12,27 +12,15 @@ from dataclasses import dataclass, field import unicodedata import time -import re import json -import uuid import warnings from datetime import datetime, timezone -import importlib.resources as ir -from contextlib import contextmanager -from contextvars import ContextVar -from urllib.parse import quote as _url_quote, parse_qs, urlparse +from urllib.parse import quote as _url_quote from ..core._http import _HttpClient from ._upload import _FileUploadMixin from ._relationships import _RelationshipOperationsMixin -from ..models.relationship import ( - LookupAttributeMetadata, - OneToManyRelationshipMetadata, - CascadeConfiguration, -) -from ..models.labels import Label, LocalizedLabel -from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK from ..core.errors import * from ._raw_request import _RawRequest from ..core._error_codes import ( @@ -48,120 +36,21 @@ METADATA_TABLE_NOT_FOUND, METADATA_TABLE_ALREADY_EXISTS, METADATA_COLUMN_NOT_FOUND, - VALIDATION_UNSUPPORTED_CACHE_KIND, ) -from .. import __version__ as _SDK_VERSION - -_USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" -_GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") -_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) -_DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) - - -def _extract_pagingcookie(next_link: str) -> Optional[str]: - """Extract the raw pagingcookie value from a SQL ``@odata.nextLink`` URL. - - The Dataverse SQL endpoint has a server-side bug where the pagingcookie - (containing first/last record GUIDs) does not advance between pages even - though ``pagenumber`` increments. Detecting a repeated cookie lets the - pagination loop break instead of looping indefinitely. - - Returns the pagingcookie string if present, or ``None`` if not found. - """ - try: - qs = parse_qs(urlparse(next_link).query) - skiptoken = qs.get("$skiptoken", [None])[0] - if not skiptoken: - return None - # parse_qs already URL-decodes the value once, giving the outer XML with - # pagingcookie still percent-encoded (e.g. pagingcookie="%3ccookie..."). - # A second decode is intentionally omitted: decoding again would turn %22 - # into " inside the cookie XML, breaking the regex and causing every page - # to extract the same truncated prefix regardless of the actual GUIDs. - m = re.search(r'pagingcookie="([^"]+)"', skiptoken) - if m: - return m.group(1) - except Exception: - pass - return None - - -@dataclass -class _RequestContext: - """Structured request context used by ``_request`` to clarify payload and metadata.""" - - method: str - url: str - expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES - headers: Optional[Dict[str, str]] = None - kwargs: Dict[str, Any] = field(default_factory=dict) - - @classmethod - def build( - cls, - method: str, - url: str, - *, - expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, - merge_headers: Optional[Callable[[Optional[Dict[str, str]]], Dict[str, str]]] = None, - **kwargs: Any, - ) -> "_RequestContext": - headers = kwargs.get("headers") - headers = merge_headers(headers) if merge_headers else (headers or {}) - headers.setdefault("x-ms-client-request-id", str(uuid.uuid4())) - headers.setdefault("x-ms-correlation-id", _CALL_SCOPE_CORRELATION_ID.get()) - kwargs["headers"] = headers - return cls( - method=method, - url=url, - expected=expected, - headers=headers, - kwargs=kwargs or {}, - ) +from ._odata_base import ( + _ODataBase, + _GUID_RE, + _extract_pagingcookie, + _USER_AGENT, + _DEFAULT_EXPECTED_STATUSES, + _RequestContext, +) -class _ODataClient(_FileUploadMixin, _RelationshipOperationsMixin): +class _ODataClient(_FileUploadMixin, _RelationshipOperationsMixin, _ODataBase): """Dataverse Web API client: CRUD, SQL-over-API, and table metadata helpers.""" - @staticmethod - def _escape_odata_quotes(value: str) -> str: - """Escape single quotes for OData queries (by doubling them).""" - return value.replace("'", "''") - - @staticmethod - def _normalize_cache_key(table_schema_name: str) -> str: - """Normalize table_schema_name to lowercase for case-insensitive cache keys.""" - return table_schema_name.lower() if isinstance(table_schema_name, str) else "" - - @staticmethod - def _lowercase_keys(record: Dict[str, Any]) -> Dict[str, Any]: - """Convert all dictionary keys to lowercase for case-insensitive column names. - - Dataverse LogicalNames for attributes are stored lowercase, but users may - provide PascalCase names (matching SchemaName). This normalizes the input. - - Keys containing ``@odata.`` (e.g. ``new_CustomerId@odata.bind``) are - preserved as-is because the navigation property portion before ``@`` - must retain its original casing (case-sensitive navigation property name). The OData - parser validates ``@odata.bind`` property names **case-sensitively** - against the entity's declared navigation properties, so lowercasing - these keys causes ``400 - undeclared property`` errors. - """ - if not isinstance(record, dict): - return record - return {k.lower() if isinstance(k, str) and "@odata." not in k else k: v for k, v in record.items()} - - @staticmethod - def _lowercase_list(items: Optional[List[str]]) -> Optional[List[str]]: - """Convert all strings in a list to lowercase for case-insensitive column names. - - Used for $select and $orderby parameters where column names must be lowercase. - """ - if not items: - return items - return [item.lower() if isinstance(item, str) else item for item in items] - def __init__( self, auth, @@ -183,22 +72,8 @@ def __init__( :type session: :class:`requests.Session` | ``None`` :raises ValueError: If ``base_url`` is empty after stripping. """ + super().__init__(base_url, config) self.auth = auth - self.base_url = (base_url or "").rstrip("/") - if not self.base_url: - raise ValueError("base_url is required.") - self.api = f"{self.base_url}/api/data/v9.2" - self.config = ( - config - or __import__( - "PowerPlatform.Dataverse.core.config", fromlist=["DataverseConfig"] - ).DataverseConfig.from_env() - ) - self._http_logger = None - if self.config.log_config is not None: - from ..core._http_logger import _HttpLogger - - self._http_logger = _HttpLogger(self.config.log_config) self._http = _HttpClient( retries=self.config.http_retries, backoff=self.config.http_backoff, @@ -206,23 +81,6 @@ def __init__( session=session, logger=self._http_logger, ) - ctx_obj = self.config.operation_context - self._operation_context = ctx_obj.user_agent_context if ctx_obj else None - self._logical_to_entityset_cache: dict[str, str] = {} - # Cache: normalized table_schema_name (lowercase) -> primary id attribute (e.g. accountid) - self._logical_primaryid_cache: dict[str, str] = {} - self._picklist_label_cache: dict[str, dict] = {} - self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL - - @contextmanager - def _call_scope(self): - """Context manager to generate a new correlation id for each SDK call scope.""" - shared_id = str(uuid.uuid4()) - token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) - try: - yield shared_id - finally: - _CALL_SCOPE_CORRELATION_ID.reset(token) def close(self) -> None: """Close the OData client and release resources. @@ -230,14 +88,9 @@ def close(self) -> None: Clears all internal caches and closes the underlying HTTP client. Safe to call multiple times. """ - self._logical_to_entityset_cache.clear() - self._logical_primaryid_cache.clear() - self._picklist_label_cache.clear() + super().close() if self._http is not None: self._http.close() - if self._http_logger is not None: - self._http_logger.close() - self._http_logger = None def _headers(self) -> Dict[str, str]: """Build standard OData headers with bearer auth.""" @@ -421,36 +274,6 @@ def _create_multiple(self, entity_set: str, table_schema_name: str, records: Lis return out return [] - def _build_alternate_key_str(self, alternate_key: Dict[str, Any]) -> str: - """Build an OData alternate key segment from a mapping of key names to values. - - String values are single-quoted and escaped; all other values are rendered as-is. - - :param alternate_key: Mapping of alternate key attribute names to their values. - Must be a non-empty dict with string keys. - :type alternate_key: ``dict[str, Any]`` - - :return: Comma-separated key=value pairs suitable for use in a URL segment. - :rtype: ``str`` - - :raises ValueError: If ``alternate_key`` is empty. - :raises TypeError: If any key in ``alternate_key`` is not a string. - """ - if not alternate_key: - raise ValueError("alternate_key must be a non-empty dict") - bad_keys = [k for k in alternate_key if not isinstance(k, str)] - if bad_keys: - raise TypeError(f"alternate_key keys must be strings; got: {bad_keys!r}") - parts = [] - for k, v in alternate_key.items(): - k_lower = k.lower() if isinstance(k, str) else k - if isinstance(v, str): - v_escaped = self._escape_odata_quotes(v) - parts.append(f"{k_lower}='{v_escaped}'") - else: - parts.append(f"{k_lower}={v}") - return ",".join(parts) - def _upsert( self, entity_set: str, @@ -622,23 +445,6 @@ def _delete_multiple( job_id = body.get("JobId") return job_id - def _format_key(self, key: str) -> str: - k = key.strip() - if k.startswith("(") and k.endswith(")"): - return k - # Escape single quotes in alternate key values - if "=" in k and "'" in k: - - def esc(match): - # match.group(1) is the key, match.group(2) is the value - return f"{match.group(1)}='{self._escape_odata_quotes(match.group(2))}'" - - k = re.sub(r"(\w+)=\'([^\']*)\'", esc, k) - return f"({k})" - if len(k) == 36 and "-" in k: - return f"({k})" - return f"({k})" - def _update(self, table_schema_name: str, key: str, data: Dict[str, Any]) -> None: """Update an existing record by GUID. @@ -689,7 +495,14 @@ def _delete(self, table_schema_name: str, key: str) -> None: """ self._execute_raw(self._build_delete(table_schema_name, key)) - def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = None) -> Dict[str, Any]: + def _get( + self, + table_schema_name: str, + key: str, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Dict[str, Any]: """Retrieve a single record. :param table_schema_name: Schema name of the table. @@ -698,11 +511,19 @@ def _get(self, table_schema_name: str, key: str, select: Optional[List[str]] = N :type key: ``str`` :param select: Columns to select; joined with commas into $select. :type select: ``list[str]`` | ``None`` + :param expand: Navigation properties to expand (``$expand``); passed as-is (case-sensitive). + :type expand: ``list[str]`` | ``None`` + :param include_annotations: OData annotation pattern for the ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: ``str`` | ``None`` :return: Retrieved record dictionary (may be empty if no selected attributes). :rtype: ``dict[str, Any]`` """ - return self._execute_raw(self._build_get(table_schema_name, key, select=select)).json() + return self._execute_raw( + self._build_get( + table_schema_name, key, select=select, expand=expand, include_annotations=include_annotations + ) + ).json() def _get_multiple( self, @@ -796,165 +617,6 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st yield [x for x in items if isinstance(x, dict)] next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") if isinstance(data, dict) else None - # ----------------------- SQL guardrail patterns -------------------- - _SQL_WRITE_RE = re.compile( - r"^\s*(?:INSERT|UPDATE|DELETE|DROP|TRUNCATE|ALTER|CREATE|EXEC|GRANT|REVOKE|BULK)\b", - re.IGNORECASE, - ) - _SQL_COMMENT_RE = re.compile(r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|--[^\n]*", re.DOTALL) - _SQL_LEADING_WILDCARD_RE = re.compile(r"\bLIKE\s+'%[^']", re.IGNORECASE) - _SQL_IMPLICIT_CROSS_JOIN_RE = re.compile( - r"\bFROM\s+[A-Za-z0-9_]+(?:\s+[A-Za-z0-9_]+)?\s*,\s*[A-Za-z0-9_]+", - re.IGNORECASE, - ) - # Server-blocked SQL patterns (save the round-trip by catching early) - _SQL_UNSUPPORTED_JOIN_RE = re.compile( - r"\b(?:CROSS\s+JOIN|RIGHT\s+(?:OUTER\s+)?JOIN|FULL\s+(?:OUTER\s+)?JOIN)\b", - re.IGNORECASE, - ) - _SQL_UNION_RE = re.compile(r"\bUNION\b", re.IGNORECASE) - _SQL_HAVING_RE = re.compile(r"\bHAVING\b", re.IGNORECASE) - _SQL_CTE_RE = re.compile(r"^\s*WITH\b", re.IGNORECASE) - _SQL_SUBQUERY_RE = re.compile( - r"\bIN\s*\(\s*SELECT\b|\bEXISTS\s*\(\s*SELECT\b|\(\s*SELECT\b.*\bFROM\b", - re.IGNORECASE, - ) - # SELECT * is intentionally rejected -- not a technical limitation but a - # deliberate design decision. Wide entities (e.g. account has 307 columns) - # make SELECT * extremely expensive on shared database infrastructure. - # COUNT(*) is NOT matched because COUNT appears before the *. - _SQL_SELECT_STAR_RE = re.compile( - r"\bSELECT\b\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?\*\s", - re.IGNORECASE, - ) - - def _sql_guardrails(self, sql: str) -> str: - """Apply safety guardrails to a SQL query before sending to the server. - - Checks split into two categories: - - **Blocked** (``ValidationError`` -- saves a server round-trip): - - 1. Write statements (INSERT/UPDATE/DELETE/DROP/etc.) - 2. CROSS JOIN, RIGHT JOIN, FULL OUTER JOIN (server rejects these) - 3. UNION / UNION ALL (server rejects) - 4. HAVING clause (server rejects) - 5. CTE / WITH clause (server rejects) - 6. Subqueries -- IN (SELECT ...), EXISTS (SELECT ...) (server rejects) - 7. SELECT * -- intentional design decision, not a technical limitation. - Wide entities make wildcard selects extremely expensive on shared - database infrastructure. ``COUNT(*)`` is not affected. - - **Warned** (``UserWarning`` -- query still executes): - - 8. Leading-wildcard LIKE (full table scan) - 9. Implicit cross join FROM a, b (cartesian product) - - All blocked patterns are also blocked by the server, but catching - them here saves the network round-trip and provides clearer error - messages. To bypass a specific check (e.g., if the server adds - support in the future), all checks are in this single method. - - :param sql: The SQL string (already stripped). - :return: The SQL string (unchanged). - :raises ValidationError: If the SQL contains a blocked pattern. - """ - # --- BLOCKED (save server round-trip) --- - - # 1. Block writes (strip SQL comments first to catch comment-prefixed writes) - sql_no_comments = self._SQL_COMMENT_RE.sub(" ", sql).strip() - if self._SQL_WRITE_RE.search(sql_no_comments): - raise ValidationError( - "SQL endpoint is read-only. Use client.records or " - "client.dataframe for write operations " - "(INSERT/UPDATE/DELETE are not supported).", - subcode=VALIDATION_SQL_WRITE_BLOCKED, - ) - - # 2. Block unsupported JOIN types - m = self._SQL_UNSUPPORTED_JOIN_RE.search(sql) - if m: - raise ValidationError( - f"Unsupported JOIN type: '{m.group(0).strip()}'. " - "Only INNER JOIN and LEFT JOIN are supported by the " - "Dataverse SQL endpoint.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 3. Block UNION - if self._SQL_UNION_RE.search(sql): - raise ValidationError( - "UNION is not supported by the Dataverse SQL endpoint. " - "Execute separate queries and combine results in Python " - "(e.g. pd.concat([df1, df2])).", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 4. Block HAVING - if self._SQL_HAVING_RE.search(sql): - raise ValidationError( - "HAVING is not supported by the Dataverse SQL endpoint. " - "Use WHERE to filter before GROUP BY instead.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 5. Block CTE / WITH - if self._SQL_CTE_RE.search(sql): - raise ValidationError( - "CTE (WITH ... AS) is not supported by the Dataverse SQL " - "endpoint. Use separate queries and combine in Python.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 6. Block subqueries - if self._SQL_SUBQUERY_RE.search(sql): - raise ValidationError( - "Subqueries are not supported by the Dataverse SQL " - "endpoint. Use separate SQL calls and combine results " - "in Python (e.g. step 1: get IDs, step 2: WHERE IN).", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # 7. Block SELECT * -- intentional design decision. - # Wide entities (e.g. account has 307 columns) make wildcard selects - # extremely expensive on shared database infrastructure. - # COUNT(*) is NOT matched: _SQL_SELECT_STAR_RE requires * to be the - # first token after SELECT/DISTINCT/TOP N, so COUNT appears before *. - if self._SQL_SELECT_STAR_RE.search(sql): - raise ValidationError( - "SELECT * is not supported. Specify column names explicitly " - "(e.g. SELECT name, revenue FROM account). " - "Use client.query.sql_columns('account') to discover available columns.", - subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, - ) - - # --- WARNED (query still executes) --- - - # 8. Warn on leading-wildcard LIKE - if self._SQL_LEADING_WILDCARD_RE.search(sql): - warnings.warn( - "Query contains a leading-wildcard LIKE pattern " - "(e.g. LIKE '%value'). This forces a full table scan " - "and may degrade performance on large tables. " - "Prefer trailing wildcards (LIKE 'value%') when possible.", - UserWarning, - stacklevel=4, - ) - - # 9. Warn on implicit cross joins (server allows but risky) - if self._SQL_IMPLICIT_CROSS_JOIN_RE.search(sql): - warnings.warn( - "Query uses an implicit cross join (FROM table1, table2). " - "This produces a cartesian product that can generate " - "millions of intermediate rows and degrade shared database " - "performance. Use explicit JOIN...ON syntax instead: " - "FROM table1 a JOIN table2 b ON a.column = b.column", - UserWarning, - stacklevel=4, - ) - - return sql - # --------------------------- SQL Custom API ------------------------- def _query_sql(self, sql: str) -> list[dict[str, Any]]: """Execute a read-only SQL SELECT using the Dataverse Web API ``?sql=`` capability. @@ -1072,25 +734,6 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]: return results - @staticmethod - def _extract_logical_table(sql: str) -> str: - """Extract the logical table name after the first standalone FROM. - - Examples: - SELECT * FROM account - SELECT col1, startfrom FROM new_sampleitem WHERE col1 = 1 - - """ - if not isinstance(sql, str): - raise ValueError("sql must be a string") - # Mask out single-quoted string literals to avoid matching FROM inside them. - masked = re.sub(r"'([^']|'')*'", "'x'", sql) - pattern = r"\bfrom\b\s+([A-Za-z0-9_]+)" # minimal, single-line regex - m = re.search(pattern, masked, flags=re.IGNORECASE) - if not m: - raise ValueError("Unable to determine table logical name from SQL (expected 'FROM ').") - return m.group(1).lower() - # ---------------------- Entity set resolution ----------------------- def _entity_set_from_schema_name(self, table_schema_name: str) -> str: """Resolve entity set name (plural) from a schema name (singular) name using metadata. @@ -1143,23 +786,6 @@ def _entity_set_from_schema_name(self, table_schema_name: str) -> str: return es # ---------------------- Table metadata helpers ---------------------- - def _label(self, text: str) -> Dict[str, Any]: - lang = int(self.config.language_code) - return { - "@odata.type": "Microsoft.Dynamics.CRM.Label", - "LocalizedLabels": [ - { - "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", - "Label": text, - "LanguageCode": lang, - } - ], - } - - def _to_pascal(self, name: str) -> str: - parts = re.split(r"[^A-Za-z0-9]+", name) - return "".join(p[:1].upper() + p[1:] for p in parts if p) - def _get_entity_by_table_schema_name( self, table_schema_name: str, @@ -1333,136 +959,6 @@ def _wait_for_attribute_visibility( f"after {total_wait} seconds (exhausted all retries)." ) from last_error - # ---------------------- Enum / Option Set helpers ------------------ - def _build_localizedlabels_payload(self, translations: Dict[int, str]) -> Dict[str, Any]: - """Build a Dataverse Label object from {: } entries. - - Ensures at least one localized label. Does not deduplicate language codes; last wins. - """ - locs: List[Dict[str, Any]] = [] - for lang, text in translations.items(): - if not isinstance(lang, int): - raise ValueError(f"Language code '{lang}' must be int") - if not isinstance(text, str) or not text.strip(): - raise ValueError(f"Label for lang {lang} must be non-empty string") - locs.append( - { - "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", - "Label": text, - "LanguageCode": lang, - } - ) - if not locs: - raise ValueError("At least one translation required") - return { - "@odata.type": "Microsoft.Dynamics.CRM.Label", - "LocalizedLabels": locs, - } - - def _enum_optionset_payload( - self, column_schema_name: str, enum_cls: type[Enum], is_primary_name: bool = False - ) -> Dict[str, Any]: - """Create local (IsGlobal=False) PicklistAttributeMetadata from an Enum subclass. - - Supports translation mapping via optional class attribute `__labels__`: - __labels__ = { 1033: { "Active": "Active", "Inactive": "Inactive" }, - 1036: { "Active": "Actif", "Inactive": "Inactif" } } - - Keys inside per-language dict may be either enum member objects or their names. - If a language lacks a label for a member, member.name is used as fallback. - The client's configured language code is always ensured to exist. - """ - all_member_items = list(enum_cls.__members__.items()) - if not all_member_items: - raise ValueError(f"Enum {enum_cls.__name__} has no members") - - # Duplicate detection - value_to_first_name: Dict[int, str] = {} - for name, member in all_member_items: - val = getattr(member, "value", None) - # Defer non-int validation to later loop for consistency - if val in value_to_first_name and value_to_first_name[val] != name: - raise ValueError( - f"Duplicate enum value {val} in {enum_cls.__name__} (names: {value_to_first_name[val]}, {name})" - ) - value_to_first_name[val] = name - - members = list(enum_cls) - # Validate integer values - for m in members: - if not isinstance(m.value, int): - raise ValueError(f"Enum member '{m.name}' has non-int value '{m.value}' (only int values supported)") - - raw_labels = getattr(enum_cls, "__labels__", None) - labels_by_lang: Dict[int, Dict[str, str]] = {} - if raw_labels is not None: - if not isinstance(raw_labels, dict): - raise ValueError("__labels__ must be a dict {lang:int -> {member: label}}") - # Build a helper map for value -> member name to resolve raw int keys - value_to_name = {m.value: m.name for m in members} - for lang, mapping in raw_labels.items(): - if not isinstance(lang, int): - raise ValueError("Language codes in __labels__ must be ints") - if not isinstance(mapping, dict): - raise ValueError(f"__labels__[{lang}] must be a dict of member names to strings") - labels_by_lang.setdefault(lang, {}) - for k, v in mapping.items(): - # Accept enum member object, its name, or raw int value (from class body reference) - if isinstance(k, enum_cls): - member_name = k.name - elif isinstance(k, int): - member_name = value_to_name.get(k) - if member_name is None: - raise ValueError(f"__labels__[{lang}] has int key {k} not matching any enum value") - else: - member_name = str(k) - if not isinstance(v, str) or not v.strip(): - raise ValueError(f"Label for {member_name} lang {lang} must be non-empty string") - labels_by_lang[lang][member_name] = v - - config_lang = int(self.config.language_code) - # Ensure config language appears (fallback to names) - all_langs = set(labels_by_lang.keys()) | {config_lang} - - options: List[Dict[str, Any]] = [] - for m in sorted(members, key=lambda x: x.value): - per_lang: Dict[int, str] = {} - for lang in all_langs: - label_text = labels_by_lang.get(lang, {}).get(m.name, m.name) - per_lang[lang] = label_text - options.append( - { - "@odata.type": "Microsoft.Dynamics.CRM.OptionMetadata", - "Value": m.value, - "Label": self._build_localizedlabels_payload(per_lang), - } - ) - - attr_label = column_schema_name.split("_")[-1] - return { - "@odata.type": "Microsoft.Dynamics.CRM.PicklistAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(attr_label), - "RequiredLevel": {"Value": "None"}, - "IsPrimaryName": bool(is_primary_name), - "OptionSet": { - "@odata.type": "Microsoft.Dynamics.CRM.OptionSetMetadata", - "IsGlobal": False, - "Options": options, - }, - } - - def _normalize_picklist_label(self, label: str) -> str: - """Normalize a label for case / diacritic insensitive comparison.""" - if not isinstance(label, str): - return "" - # Strip accents - norm = unicodedata.normalize("NFD", label) - norm = "".join(c for c in norm if unicodedata.category(c) != "Mn") - # Collapse whitespace, lowercase - norm = re.sub(r"\s+", " ", norm).strip().lower() - return norm - def _request_metadata_with_retry(self, method: str, url: str, **kwargs): """Fetch metadata with retries on transient errors.""" max_attempts = 5 @@ -1576,105 +1072,6 @@ def _convert_labels_to_ints(self, table_schema_name: str, record: Dict[str, Any] resolved_record[k] = val return resolved_record - def _attribute_payload( - self, column_schema_name: str, dtype: Any, *, is_primary_name: bool = False - ) -> Optional[Dict[str, Any]]: - # Enum-based local option set support - if isinstance(dtype, type) and issubclass(dtype, Enum): - return self._enum_optionset_payload(column_schema_name, dtype, is_primary_name=is_primary_name) - if not isinstance(dtype, str): - raise ValueError( - f"Unsupported column spec type for '{column_schema_name}': {type(dtype)} (expected str or Enum subclass)" - ) - dtype_l = dtype.lower().strip() - label = column_schema_name.split("_")[-1] - if dtype_l in ("string", "text"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.StringAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MaxLength": 200, - "FormatName": {"Value": "Text"}, - "IsPrimaryName": bool(is_primary_name), - } - if dtype_l in ("memo", "multiline"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.MemoAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MaxLength": 4000, - "FormatName": {"Value": "Text"}, - "ImeMode": "Auto", - } - if dtype_l in ("int", "integer"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.IntegerAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "Format": "None", - "MinValue": -2147483648, - "MaxValue": 2147483647, - } - if dtype_l in ("decimal", "money"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DecimalAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MinValue": -100000000000.0, - "MaxValue": 100000000000.0, - "Precision": 2, - } - if dtype_l in ("float", "double"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DoubleAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "MinValue": -100000000000.0, - "MaxValue": 100000000000.0, - "Precision": 2, - } - if dtype_l in ("datetime", "date"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "Format": "DateOnly", - "ImeMode": "Inactive", - } - if dtype_l in ("bool", "boolean"): - return { - "@odata.type": "Microsoft.Dynamics.CRM.BooleanAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - "OptionSet": { - "@odata.type": "Microsoft.Dynamics.CRM.BooleanOptionSetMetadata", - "TrueOption": { - "Value": 1, - "Label": self._label("True"), - }, - "FalseOption": { - "Value": 0, - "Label": self._label("False"), - }, - "IsGlobal": False, - }, - } - if dtype_l == "file": - return { - "@odata.type": "Microsoft.Dynamics.CRM.FileAttributeMetadata", - "SchemaName": column_schema_name, - "DisplayName": self._label(label), - "RequiredLevel": {"Value": "None"}, - } - return None - def _get_table_info(self, table_schema_name: str) -> Optional[Dict[str, Any]]: """Return basic metadata for a custom table if it exists. @@ -2314,217 +1711,64 @@ def _build_get( record_id: str, *, select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, ) -> _RawRequest: """Build a single-record GET request without sending it.""" entity_set = self._entity_set_from_schema_name(table) - url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + params: List[str] = [] if select: - url += "?$select=" + ",".join(self._lowercase_list(select)) - return _RawRequest(method="GET", url=url) + params.append("$select=" + ",".join(self._lowercase_list(select))) + if expand: + params.append("$expand=" + ",".join(expand)) + url = f"{self.api}/{entity_set}{self._format_key(record_id)}" + if params: + url += "?" + "&".join(params) + headers = None + if include_annotations: + headers = {"Prefer": f'odata.include-annotations="{include_annotations}"'} + return _RawRequest(method="GET", url=url, headers=headers) - def _build_create_entity( + def _build_list( self, table: str, - columns: Dict[str, Any], - solution: Optional[str] = None, - primary_column: Optional[str] = None, - display_name: Optional[str] = None, - ) -> _RawRequest: - """Build an EntityDefinitions POST request without sending it.""" - if primary_column: - primary_attr = primary_column - else: - primary_attr = f"{table.split('_', 1)[0]}_Name" if "_" in table else "new_Name" - attributes = [self._attribute_payload(primary_attr, "string", is_primary_name=True)] - for col_name, dtype in columns.items(): - attr = self._attribute_payload(col_name, dtype) - if not attr: - raise ValidationError( - f"Unsupported column type '{dtype}' for column '{col_name}'.", - subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, - ) - attributes.append(attr) - if display_name is not None: - if not isinstance(display_name, str) or not display_name.strip(): - raise TypeError("display_name must be a non-empty string when provided") - label = display_name if display_name is not None else table - body = { - "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", - "SchemaName": table, - "DisplayName": self._label(label), - "DisplayCollectionName": self._label(label + "s"), - "Description": self._label(f"Custom entity for {label}"), - "OwnershipType": "UserOwned", - "HasActivities": False, - "HasNotes": True, - "IsActivity": False, - "Attributes": attributes, - } - url = f"{self.api}/EntityDefinitions" - if solution: - url += f"?SolutionUniqueName={solution}" - return _RawRequest( - method="POST", - url=url, - body=json.dumps(body, ensure_ascii=False), - ) - - def _build_delete_entity(self, metadata_id: str) -> _RawRequest: - """Build an EntityDefinitions DELETE request without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/EntityDefinitions({metadata_id})", - headers={"If-Match": "*"}, - ) - - def _build_get_entity(self, table: str) -> _RawRequest: - """Build an EntityDefinitions GET request without sending it.""" - logical = self._escape_odata_quotes(table.lower()) - return _RawRequest( - method="GET", - url=( - f"{self.api}/EntityDefinitions" - f"?$select=MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute" - f"&$filter=LogicalName eq '{logical}'" - ), - ) - - def _build_list_entities( - self, *, - filter: Optional[str] = None, select: Optional[List[str]] = None, + filter: Optional[str] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, ) -> _RawRequest: - """Build an EntityDefinitions list GET request without sending it.""" - base_filter = "IsPrivate eq false" - if filter: - combined_filter = f"{base_filter} and ({filter})" - else: - combined_filter = base_filter - url = f"{self.api}/EntityDefinitions?$filter={combined_filter}" - if select is not None and isinstance(select, str): - raise TypeError("select must be a list of property names, not a bare string") + """Build a multi-record GET request (single page, no pagination) without sending it.""" + entity_set = self._entity_set_from_schema_name(table) + params: List[str] = [] if select: - url += "&$select=" + ",".join(select) - return _RawRequest(method="GET", url=url) - - def _build_create_column( - self, - entity_metadata_id: str, - col_name: str, - dtype: Any, - ) -> _RawRequest: - """Build an Attributes POST request for one column without sending it.""" - attr = self._attribute_payload(col_name, dtype) - if not attr: - raise ValidationError( - f"Unsupported column type '{dtype}' for column '{col_name}'.", - subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, - ) - return _RawRequest( - method="POST", - url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes", - body=json.dumps(attr, ensure_ascii=False), - ) - - def _build_delete_column( - self, - entity_metadata_id: str, - col_metadata_id: str, - ) -> _RawRequest: - """Build an Attributes DELETE request for one column without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes({col_metadata_id})", - headers={"If-Match": "*"}, - ) - - @staticmethod - def _build_lookup_field_models( - referencing_table: str, - lookup_field_name: str, - referenced_table: str, - *, - display_name: Optional[str] = None, - description: Optional[str] = None, - required: bool = False, - cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, - language_code: int = 1033, - ) -> tuple: - """Build a (lookup, relationship) pair for a lookup field creation. - - Returns ``(LookupAttributeMetadata, OneToManyRelationshipMetadata)``. - Used by both the batch resolver and ``TableOperations.create_lookup_field`` - to avoid duplicating the metadata assembly logic. - - Note: ``referencing_table`` and ``referenced_table`` are lowercased - automatically because Dataverse stores entity logical names in - lowercase. ``lookup_field_name`` is kept as-is (it is a SchemaName). - """ - # Dataverse logical names are always lowercase. Callers may pass - # SchemaName-cased values (e.g. "new_SQLTeam"); normalise here so - # the relationship metadata uses valid logical names. - referencing_lower = referencing_table.lower() - referenced_lower = referenced_table.lower() - - lookup = LookupAttributeMetadata( - schema_name=lookup_field_name, - display_name=Label( - localized_labels=[ - LocalizedLabel( - label=display_name or referenced_table, - language_code=language_code, - ) - ] - ), - required_level="ApplicationRequired" if required else "None", - ) - if description: - lookup.description = Label( - localized_labels=[LocalizedLabel(label=description, language_code=language_code)] - ) - rel_name = f"{referenced_lower}_{referencing_lower}_{lookup_field_name}" - relationship = OneToManyRelationshipMetadata( - schema_name=rel_name, - referenced_entity=referenced_lower, - referencing_entity=referencing_lower, - referenced_attribute=f"{referenced_lower}id", - cascade_configuration=CascadeConfiguration(delete=cascade_delete), - ) - return lookup, relationship - - def _build_create_relationship( - self, - body: Dict[str, Any], - *, - solution: Optional[str] = None, - ) -> _RawRequest: - """Build a RelationshipDefinitions POST request without sending it.""" - headers: Dict[str, str] = {} - if solution: - headers["MSCRM.SolutionUniqueName"] = solution - return _RawRequest( - method="POST", - url=f"{self.api}/RelationshipDefinitions", - body=json.dumps(body, ensure_ascii=False), - headers=headers or None, - ) - - def _build_delete_relationship(self, relationship_id: str) -> _RawRequest: - """Build a RelationshipDefinitions DELETE request without sending it.""" - return _RawRequest( - method="DELETE", - url=f"{self.api}/RelationshipDefinitions({relationship_id})", - headers={"If-Match": "*"}, - ) - - def _build_get_relationship(self, schema_name: str) -> _RawRequest: - """Build a RelationshipDefinitions GET request without sending it.""" - escaped = self._escape_odata_quotes(schema_name) - return _RawRequest( - method="GET", - url=f"{self.api}/RelationshipDefinitions?$filter=SchemaName eq '{escaped}'", - ) + params.append("$select=" + ",".join(self._lowercase_list(select))) + if filter: + params.append("$filter=" + filter) + if orderby: + params.append("$orderby=" + ",".join(orderby)) + if top is not None: + params.append(f"$top={top}") + if expand: + params.append("$expand=" + ",".join(expand)) + if count: + params.append("$count=true") + url = f"{self.api}/{entity_set}" + if params: + url += "?" + "&".join(params) + prefer_parts: List[str] = [] + if page_size is not None: + ps = int(page_size) + if ps > 0: + prefer_parts.append(f"odata.maxpagesize={ps}") + if include_annotations: + prefer_parts.append(f'odata.include-annotations="{include_annotations}"') + headers = {"Prefer": ",".join(prefer_parts)} if prefer_parts else None + return _RawRequest(method="GET", url=url, headers=headers) def _build_sql(self, sql: str) -> _RawRequest: """Build a SQL query GET request without sending it. @@ -2547,27 +1791,3 @@ def _build_sql(self, sql: str) -> _RawRequest: method="GET", url=f"{self.api}/{entity_set}?sql={_url_quote(sql, safe='')}", ) - - # ---------------------- Cache maintenance ------------------------- - def _flush_cache( - self, - kind, - ) -> int: - """Flush cached client metadata/state. - - :param kind: Cache kind to flush (only ``"picklist"`` supported). - :type kind: ``str`` - :return: Number of cache entries removed. - :rtype: ``int`` - :raises ValidationError: If ``kind`` is unsupported. - """ - k = (kind or "").strip().lower() - if k != "picklist": - raise ValidationError( - f"Unsupported cache kind '{kind}' (only 'picklist' is implemented)", - subcode=VALIDATION_UNSUPPORTED_CACHE_KIND, - ) - - removed = len(self._picklist_label_cache) - self._picklist_label_cache.clear() - return removed diff --git a/src/PowerPlatform/Dataverse/data/_odata_base.py b/src/PowerPlatform/Dataverse/data/_odata_base.py new file mode 100644 index 00000000..34ff7c55 --- /dev/null +++ b/src/PowerPlatform/Dataverse/data/_odata_base.py @@ -0,0 +1,936 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared pure-logic base for the Dataverse OData client. Contains no I/O. + +Subclasses add the HTTP transport layer (sync or async) while sharing all +URL construction, payload building, cache helpers, and other stateless logic. +""" + +from __future__ import annotations + +import json +import re +import unicodedata +import uuid +import warnings +from contextlib import contextmanager +from contextvars import ContextVar +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, Callable, Dict, List, Optional, Union +from urllib.parse import parse_qs, urlparse + +from .. import __version__ as _SDK_VERSION + +from ..core.errors import ValidationError +from ..core._error_codes import ( + VALIDATION_UNSUPPORTED_COLUMN_TYPE, + VALIDATION_UNSUPPORTED_CACHE_KIND, + VALIDATION_SQL_WRITE_BLOCKED, + VALIDATION_SQL_UNSUPPORTED_SYNTAX, +) +from ..models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + CascadeConfiguration, +) +from ..models.labels import Label, LocalizedLabel +from ..common.constants import CASCADE_BEHAVIOR_REMOVE_LINK +from ._raw_request import _RawRequest + +__all__ = [] + +_GUID_RE = re.compile(r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}") +_CALL_SCOPE_CORRELATION_ID: ContextVar[Optional[str]] = ContextVar("_CALL_SCOPE_CORRELATION_ID", default=None) +_USER_AGENT = f"DataverseSvcPythonClient:{_SDK_VERSION}" +_DEFAULT_EXPECTED_STATUSES: tuple[int, ...] = (200, 201, 202, 204) + + +def _extract_pagingcookie(next_link: str) -> Optional[str]: + """Extract the raw pagingcookie value from a SQL ``@odata.nextLink`` URL. + + The Dataverse SQL endpoint has a server-side bug where the pagingcookie + (containing first/last record GUIDs) does not advance between pages even + though ``pagenumber`` increments. Detecting a repeated cookie lets the + pagination loop break instead of looping indefinitely. + + Returns the pagingcookie string if present, or ``None`` if not found. + """ + try: + qs = parse_qs(urlparse(next_link).query) + skiptoken = qs.get("$skiptoken", [None])[0] + if not skiptoken: + return None + # parse_qs already URL-decodes the value once, giving the outer XML with + # pagingcookie still percent-encoded (e.g. pagingcookie="%3ccookie..."). + # A second decode is intentionally omitted: decoding again would turn %22 + # into " inside the cookie XML, breaking the regex and causing every page + # to extract the same truncated prefix regardless of the actual GUIDs. + m = re.search(r'pagingcookie="([^"]+)"', skiptoken) + if m: + return m.group(1) + except Exception: + pass + return None + + +@dataclass +class _RequestContext: + """Structured request context used by ``_request`` to clarify payload and metadata.""" + + method: str + url: str + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES + headers: Optional[Dict[str, str]] = None + kwargs: Dict[str, Any] = field(default_factory=dict) + + @classmethod + def build( + cls, + method: str, + url: str, + *, + expected: tuple[int, ...] = _DEFAULT_EXPECTED_STATUSES, + merge_headers: Optional[Callable[[Optional[Dict[str, str]]], Dict[str, str]]] = None, + **kwargs: Any, + ) -> "_RequestContext": + headers = kwargs.get("headers") + headers = merge_headers(headers) if merge_headers else (headers or {}) + headers.setdefault("x-ms-client-request-id", str(uuid.uuid4())) + headers.setdefault("x-ms-correlation-id", _CALL_SCOPE_CORRELATION_ID.get()) + kwargs["headers"] = headers + return cls( + method=method, + url=url, + expected=expected, + headers=headers, + kwargs=kwargs or {}, + ) + + +class _ODataBase: + """Pure-logic base for the Dataverse OData client. + + Provides URL construction, cache management, payload builders, and other + stateless or cache-only helpers. No I/O is performed here; subclasses + must supply ``_request`` and the rest of the HTTP transport layer. + """ + + def __init__(self, base_url: str, config=None) -> None: + """Initialise shared state: URL, API root, config, in-memory caches, and HTTP logger. + + :param base_url: Organisation base URL (e.g. ``"https://.crm.dynamics.com"``). + :type base_url: :class:`str` + :param config: Optional Dataverse configuration (HTTP retry, backoff, timeout, + language code, HTTP diagnostic logging). If omitted, ``DataverseConfig.from_env()`` is used. + :type config: ~PowerPlatform.Dataverse.core.config.DataverseConfig or None + :raises ValueError: If ``base_url`` is empty after stripping. + """ + self.base_url = (base_url or "").rstrip("/") + if not self.base_url: + raise ValueError("base_url is required.") + self.api = f"{self.base_url}/api/data/v9.2" + self.config = ( + config + or __import__( + "PowerPlatform.Dataverse.core.config", fromlist=["DataverseConfig"] + ).DataverseConfig.from_env() + ) + self._logical_to_entityset_cache: dict[str, str] = {} + # Cache: normalized table_schema_name (lowercase) -> primary id attribute (e.g. accountid) + self._logical_primaryid_cache: dict[str, str] = {} + self._picklist_label_cache: dict[str, dict] = {} + self._picklist_cache_ttl_seconds = 3600 # 1 hour TTL + ctx_obj = self.config.operation_context + self._operation_context: Optional[str] = ctx_obj.user_agent_context if ctx_obj else None + self._http_logger = None + if self.config.log_config is not None: + from ..core._http_logger import _HttpLogger + + self._http_logger = _HttpLogger(self.config.log_config) + + # ------------------------------------------------------------------ + # Static helpers + # ------------------------------------------------------------------ + + @staticmethod + def _escape_odata_quotes(value: str) -> str: + """Escape single quotes for OData queries (by doubling them).""" + return value.replace("'", "''") + + @staticmethod + def _normalize_cache_key(table_schema_name: str) -> str: + """Normalize table_schema_name to lowercase for case-insensitive cache keys.""" + return table_schema_name.lower() if isinstance(table_schema_name, str) else "" + + @staticmethod + def _lowercase_keys(record: Dict[str, Any]) -> Dict[str, Any]: + """Convert all dictionary keys to lowercase for case-insensitive column names. + + Dataverse LogicalNames for attributes are stored lowercase, but users may + provide PascalCase names (matching SchemaName). This normalizes the input. + + Keys containing ``@odata.`` (e.g. ``new_CustomerId@odata.bind``) are + preserved as-is because the navigation property portion before ``@`` + must retain its original casing (case-sensitive navigation property name). The OData + parser validates ``@odata.bind`` property names **case-sensitively** + against the entity's declared navigation properties, so lowercasing + these keys causes ``400 - undeclared property`` errors. + """ + if not isinstance(record, dict): + return record + return {k.lower() if isinstance(k, str) and "@odata." not in k else k: v for k, v in record.items()} + + @staticmethod + def _lowercase_list(items: Optional[List[str]]) -> Optional[List[str]]: + """Convert all strings in a list to lowercase for case-insensitive column names. + + Used for $select and $orderby parameters where column names must be lowercase. + """ + if not items: + return items + return [item.lower() if isinstance(item, str) else item for item in items] + + @staticmethod + def _extract_logical_table(sql: str) -> str: + """Extract the logical table name after the first standalone FROM. + + Examples: + SELECT * FROM account + SELECT col1, startfrom FROM new_sampleitem WHERE col1 = 1 + + """ + if not isinstance(sql, str): + raise ValueError("sql must be a string") + # Mask out single-quoted string literals to avoid matching FROM inside them. + masked = re.sub(r"'([^']|'')*'", "'x'", sql) + pattern = r"\bfrom\b\s+([A-Za-z0-9_]+)" # minimal, single-line regex + m = re.search(pattern, masked, flags=re.IGNORECASE) + if not m: + raise ValueError("Unable to determine table logical name from SQL (expected 'FROM ').") + return m.group(1).lower() + + # ------------------------------------------------------------------ + # Instance helpers + # ------------------------------------------------------------------ + + @contextmanager + def _call_scope(self): + """Context manager to generate a new correlation id for each SDK call scope.""" + shared_id = str(uuid.uuid4()) + token = _CALL_SCOPE_CORRELATION_ID.set(shared_id) + try: + yield shared_id + finally: + _CALL_SCOPE_CORRELATION_ID.reset(token) + + def _format_key(self, key: str) -> str: + k = key.strip() + if k.startswith("(") and k.endswith(")"): + return k + # Escape single quotes in alternate key values + if "=" in k and "'" in k: + + def esc(match): + # match.group(1) is the key, match.group(2) is the value + return f"{match.group(1)}='{self._escape_odata_quotes(match.group(2))}'" + + k = re.sub(r"(\w+)=\'([^\']*)\'", esc, k) + return f"({k})" + if len(k) == 36 and "-" in k: + return f"({k})" + return f"({k})" + + def _build_alternate_key_str(self, alternate_key: Dict[str, Any]) -> str: + """Build an OData alternate key segment from a mapping of key names to values. + + String values are single-quoted and escaped; all other values are rendered as-is. + + :param alternate_key: Mapping of alternate key attribute names to their values. + Must be a non-empty dict with string keys. + :type alternate_key: ``dict[str, Any]`` + + :return: Comma-separated key=value pairs suitable for use in a URL segment. + :rtype: ``str`` + + :raises ValueError: If ``alternate_key`` is empty. + :raises TypeError: If any key in ``alternate_key`` is not a string. + """ + if not alternate_key: + raise ValueError("alternate_key must be a non-empty dict") + bad_keys = [k for k in alternate_key if not isinstance(k, str)] + if bad_keys: + raise TypeError(f"alternate_key keys must be strings; got: {bad_keys!r}") + parts = [] + for k, v in alternate_key.items(): + k_lower = k.lower() if isinstance(k, str) else k + if isinstance(v, str): + v_escaped = self._escape_odata_quotes(v) + parts.append(f"{k_lower}='{v_escaped}'") + else: + parts.append(f"{k_lower}={v}") + return ",".join(parts) + + def _label(self, text: str) -> Dict[str, Any]: + lang = int(self.config.language_code) + return { + "@odata.type": "Microsoft.Dynamics.CRM.Label", + "LocalizedLabels": [ + { + "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", + "Label": text, + "LanguageCode": lang, + } + ], + } + + def _to_pascal(self, name: str) -> str: + parts = re.split(r"[^A-Za-z0-9]+", name) + return "".join(p[:1].upper() + p[1:] for p in parts if p) + + def _normalize_picklist_label(self, label: str) -> str: + """Normalize a label for case / diacritic insensitive comparison.""" + if not isinstance(label, str): + return "" + # Strip accents + norm = unicodedata.normalize("NFD", label) + norm = "".join(c for c in norm if unicodedata.category(c) != "Mn") + # Collapse whitespace, lowercase + norm = re.sub(r"\s+", " ", norm).strip().lower() + return norm + + # ------------------------------------------------------------------ + # Payload builders (no I/O) + # ------------------------------------------------------------------ + + def _build_localizedlabels_payload(self, translations: Dict[int, str]) -> Dict[str, Any]: + """Build a Dataverse Label object from {: } entries. + + Ensures at least one localized label. Does not deduplicate language codes; last wins. + """ + locs: List[Dict[str, Any]] = [] + for lang, text in translations.items(): + if not isinstance(lang, int): + raise ValueError(f"Language code '{lang}' must be int") + if not isinstance(text, str) or not text.strip(): + raise ValueError(f"Label for lang {lang} must be non-empty string") + locs.append( + { + "@odata.type": "Microsoft.Dynamics.CRM.LocalizedLabel", + "Label": text, + "LanguageCode": lang, + } + ) + if not locs: + raise ValueError("At least one translation required") + return { + "@odata.type": "Microsoft.Dynamics.CRM.Label", + "LocalizedLabels": locs, + } + + def _enum_optionset_payload( + self, column_schema_name: str, enum_cls: type[Enum], is_primary_name: bool = False + ) -> Dict[str, Any]: + """Create local (IsGlobal=False) PicklistAttributeMetadata from an Enum subclass. + + Supports translation mapping via optional class attribute `__labels__`: + __labels__ = { 1033: { "Active": "Active", "Inactive": "Inactive" }, + 1036: { "Active": "Actif", "Inactive": "Inactif" } } + + Keys inside per-language dict may be either enum member objects or their names. + If a language lacks a label for a member, member.name is used as fallback. + The client's configured language code is always ensured to exist. + """ + all_member_items = list(enum_cls.__members__.items()) + if not all_member_items: + raise ValueError(f"Enum {enum_cls.__name__} has no members") + + # Duplicate detection + value_to_first_name: Dict[int, str] = {} + for name, member in all_member_items: + val = getattr(member, "value", None) + # Defer non-int validation to later loop for consistency + if val in value_to_first_name and value_to_first_name[val] != name: + raise ValueError( + f"Duplicate enum value {val} in {enum_cls.__name__} (names: {value_to_first_name[val]}, {name})" + ) + value_to_first_name[val] = name + + members = list(enum_cls) + # Validate integer values + for m in members: + if not isinstance(m.value, int): + raise ValueError(f"Enum member '{m.name}' has non-int value '{m.value}' (only int values supported)") + + raw_labels = getattr(enum_cls, "__labels__", None) + labels_by_lang: Dict[int, Dict[str, str]] = {} + if raw_labels is not None: + if not isinstance(raw_labels, dict): + raise ValueError("__labels__ must be a dict {lang:int -> {member: label}}") + # Build a helper map for value -> member name to resolve raw int keys + value_to_name = {m.value: m.name for m in members} + for lang, mapping in raw_labels.items(): + if not isinstance(lang, int): + raise ValueError("Language codes in __labels__ must be ints") + if not isinstance(mapping, dict): + raise ValueError(f"__labels__[{lang}] must be a dict of member names to strings") + labels_by_lang.setdefault(lang, {}) + for k, v in mapping.items(): + # Accept enum member object, its name, or raw int value (from class body reference) + if isinstance(k, enum_cls): + member_name = k.name + elif isinstance(k, int): + member_name = value_to_name.get(k) + if member_name is None: + raise ValueError(f"__labels__[{lang}] has int key {k} not matching any enum value") + else: + member_name = str(k) + if not isinstance(v, str) or not v.strip(): + raise ValueError(f"Label for {member_name} lang {lang} must be non-empty string") + labels_by_lang[lang][member_name] = v + + config_lang = int(self.config.language_code) + # Ensure config language appears (fallback to names) + all_langs = set(labels_by_lang.keys()) | {config_lang} + + options: List[Dict[str, Any]] = [] + for m in sorted(members, key=lambda x: x.value): + per_lang: Dict[int, str] = {} + for lang in all_langs: + label_text = labels_by_lang.get(lang, {}).get(m.name, m.name) + per_lang[lang] = label_text + options.append( + { + "@odata.type": "Microsoft.Dynamics.CRM.OptionMetadata", + "Value": m.value, + "Label": self._build_localizedlabels_payload(per_lang), + } + ) + + attr_label = column_schema_name.split("_")[-1] + return { + "@odata.type": "Microsoft.Dynamics.CRM.PicklistAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(attr_label), + "RequiredLevel": {"Value": "None"}, + "IsPrimaryName": bool(is_primary_name), + "OptionSet": { + "@odata.type": "Microsoft.Dynamics.CRM.OptionSetMetadata", + "IsGlobal": False, + "Options": options, + }, + } + + def _attribute_payload( + self, column_schema_name: str, dtype: Any, *, is_primary_name: bool = False + ) -> Optional[Dict[str, Any]]: + # Enum-based local option set support + if isinstance(dtype, type) and issubclass(dtype, Enum): + return self._enum_optionset_payload(column_schema_name, dtype, is_primary_name=is_primary_name) + if not isinstance(dtype, str): + raise ValueError( + f"Unsupported column spec type for '{column_schema_name}': {type(dtype)} (expected str or Enum subclass)" + ) + dtype_l = dtype.lower().strip() + label = column_schema_name.split("_")[-1] + if dtype_l in ("string", "text"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.StringAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MaxLength": 200, + "FormatName": {"Value": "Text"}, + "IsPrimaryName": bool(is_primary_name), + } + if dtype_l in ("memo", "multiline"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.MemoAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MaxLength": 4000, + "FormatName": {"Value": "Text"}, + "ImeMode": "Auto", + } + if dtype_l in ("int", "integer"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.IntegerAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "Format": "None", + "MinValue": -2147483648, + "MaxValue": 2147483647, + } + if dtype_l in ("decimal", "money"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DecimalAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MinValue": -100000000000.0, + "MaxValue": 100000000000.0, + "Precision": 2, + } + if dtype_l in ("float", "double"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DoubleAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "MinValue": -100000000000.0, + "MaxValue": 100000000000.0, + "Precision": 2, + } + if dtype_l in ("datetime", "date"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.DateTimeAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "Format": "DateOnly", + "ImeMode": "Inactive", + } + if dtype_l in ("bool", "boolean"): + return { + "@odata.type": "Microsoft.Dynamics.CRM.BooleanAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + "OptionSet": { + "@odata.type": "Microsoft.Dynamics.CRM.BooleanOptionSetMetadata", + "TrueOption": { + "Value": 1, + "Label": self._label("True"), + }, + "FalseOption": { + "Value": 0, + "Label": self._label("False"), + }, + "IsGlobal": False, + }, + } + if dtype_l == "file": + return { + "@odata.type": "Microsoft.Dynamics.CRM.FileAttributeMetadata", + "SchemaName": column_schema_name, + "DisplayName": self._label(label), + "RequiredLevel": {"Value": "None"}, + } + return None + + # ------------------------------------------------------------------ + # Entity / column / relationship _build_* methods (no I/O) + # ------------------------------------------------------------------ + + def _build_create_entity( + self, + table: str, + columns: Dict[str, Any], + solution: Optional[str] = None, + primary_column: Optional[str] = None, + display_name: Optional[str] = None, + ) -> _RawRequest: + """Build an EntityDefinitions POST request without sending it.""" + if primary_column: + primary_attr = primary_column + else: + primary_attr = f"{table.split('_', 1)[0]}_Name" if "_" in table else "new_Name" + attributes = [self._attribute_payload(primary_attr, "string", is_primary_name=True)] + for col_name, dtype in columns.items(): + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + attributes.append(attr) + if display_name is not None: + if not isinstance(display_name, str) or not display_name.strip(): + raise TypeError("display_name must be a non-empty string when provided") + label = display_name if display_name is not None else table + body = { + "@odata.type": "Microsoft.Dynamics.CRM.EntityMetadata", + "SchemaName": table, + "DisplayName": self._label(label), + "DisplayCollectionName": self._label(label + "s"), + "Description": self._label(f"Custom entity for {label}"), + "OwnershipType": "UserOwned", + "HasActivities": False, + "HasNotes": True, + "IsActivity": False, + "Attributes": attributes, + } + url = f"{self.api}/EntityDefinitions" + if solution: + url += f"?SolutionUniqueName={solution}" + return _RawRequest( + method="POST", + url=url, + body=json.dumps(body, ensure_ascii=False), + ) + + def _build_delete_entity(self, metadata_id: str) -> _RawRequest: + """Build an EntityDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({metadata_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_entity(self, table: str) -> _RawRequest: + """Build an EntityDefinitions GET request without sending it.""" + logical = self._escape_odata_quotes(table.lower()) + return _RawRequest( + method="GET", + url=( + f"{self.api}/EntityDefinitions" + f"?$select=MetadataId,LogicalName,SchemaName,EntitySetName,PrimaryNameAttribute,PrimaryIdAttribute" + f"&$filter=LogicalName eq '{logical}'" + ), + ) + + def _build_list_entities( + self, + *, + filter: Optional[str] = None, + select: Optional[List[str]] = None, + ) -> _RawRequest: + """Build an EntityDefinitions list GET request without sending it.""" + base_filter = "IsPrivate eq false" + if filter: + combined_filter = f"{base_filter} and ({filter})" + else: + combined_filter = base_filter + url = f"{self.api}/EntityDefinitions?$filter={combined_filter}" + if select is not None and isinstance(select, str): + raise TypeError("select must be a list of property names, not a bare string") + if select: + url += "&$select=" + ",".join(select) + return _RawRequest(method="GET", url=url) + + def _build_create_column( + self, + entity_metadata_id: str, + col_name: str, + dtype: Any, + ) -> _RawRequest: + """Build an Attributes POST request for one column without sending it.""" + attr = self._attribute_payload(col_name, dtype) + if not attr: + raise ValidationError( + f"Unsupported column type '{dtype}' for column '{col_name}'.", + subcode=VALIDATION_UNSUPPORTED_COLUMN_TYPE, + ) + return _RawRequest( + method="POST", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes", + body=json.dumps(attr, ensure_ascii=False), + ) + + def _build_delete_column( + self, + entity_metadata_id: str, + col_metadata_id: str, + ) -> _RawRequest: + """Build an Attributes DELETE request for one column without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/EntityDefinitions({entity_metadata_id})/Attributes({col_metadata_id})", + headers={"If-Match": "*"}, + ) + + @staticmethod + def _build_lookup_field_models( + referencing_table: str, + lookup_field_name: str, + referenced_table: str, + *, + display_name: Optional[str] = None, + description: Optional[str] = None, + required: bool = False, + cascade_delete: str = CASCADE_BEHAVIOR_REMOVE_LINK, + language_code: int = 1033, + ) -> tuple: + """Build a (lookup, relationship) pair for a lookup field creation. + + Returns ``(LookupAttributeMetadata, OneToManyRelationshipMetadata)``. + Used by both the batch resolver and ``TableOperations.create_lookup_field`` + to avoid duplicating the metadata assembly logic. + + Note: ``referencing_table`` and ``referenced_table`` are lowercased + automatically because Dataverse stores entity logical names in + lowercase. ``lookup_field_name`` is kept as-is (it is a SchemaName). + """ + # Dataverse logical names are always lowercase. Callers may pass + # SchemaName-cased values (e.g. "new_SQLTeam"); normalise here so + # the relationship metadata uses valid logical names. + referencing_lower = referencing_table.lower() + referenced_lower = referenced_table.lower() + + lookup = LookupAttributeMetadata( + schema_name=lookup_field_name, + display_name=Label( + localized_labels=[ + LocalizedLabel( + label=display_name or referenced_table, + language_code=language_code, + ) + ] + ), + required_level="ApplicationRequired" if required else "None", + ) + if description: + lookup.description = Label( + localized_labels=[LocalizedLabel(label=description, language_code=language_code)] + ) + rel_name = f"{referenced_lower}_{referencing_lower}_{lookup_field_name}" + relationship = OneToManyRelationshipMetadata( + schema_name=rel_name, + referenced_entity=referenced_lower, + referencing_entity=referencing_lower, + referenced_attribute=f"{referenced_lower}id", + cascade_configuration=CascadeConfiguration(delete=cascade_delete), + ) + return lookup, relationship + + def _build_create_relationship( + self, + body: Dict[str, Any], + *, + solution: Optional[str] = None, + ) -> _RawRequest: + """Build a RelationshipDefinitions POST request without sending it.""" + headers: Dict[str, str] = {} + if solution: + headers["MSCRM.SolutionUniqueName"] = solution + return _RawRequest( + method="POST", + url=f"{self.api}/RelationshipDefinitions", + body=json.dumps(body, ensure_ascii=False), + headers=headers or None, + ) + + def _build_delete_relationship(self, relationship_id: str) -> _RawRequest: + """Build a RelationshipDefinitions DELETE request without sending it.""" + return _RawRequest( + method="DELETE", + url=f"{self.api}/RelationshipDefinitions({relationship_id})", + headers={"If-Match": "*"}, + ) + + def _build_get_relationship(self, schema_name: str) -> _RawRequest: + """Build a RelationshipDefinitions GET request without sending it.""" + escaped = self._escape_odata_quotes(schema_name) + return _RawRequest( + method="GET", + url=f"{self.api}/RelationshipDefinitions?$filter=SchemaName eq '{escaped}'", + ) + + # ------------------------------------------------------------------ + # SQL guardrails + # ------------------------------------------------------------------ + + # ----------------------- SQL guardrail patterns -------------------- + _SQL_WRITE_RE = re.compile( + r"^\s*(?:INSERT|UPDATE|DELETE|DROP|TRUNCATE|ALTER|CREATE|EXEC|GRANT|REVOKE|BULK)\b", + re.IGNORECASE, + ) + _SQL_COMMENT_RE = re.compile(r"/\*[^*]*\*+(?:[^/*][^*]*\*+)*/|--[^\n]*", re.DOTALL) + _SQL_LEADING_WILDCARD_RE = re.compile(r"\bLIKE\s+'%[^']", re.IGNORECASE) + _SQL_IMPLICIT_CROSS_JOIN_RE = re.compile( + r"\bFROM\s+[A-Za-z0-9_]+(?:\s+[A-Za-z0-9_]+)?\s*,\s*[A-Za-z0-9_]+", + re.IGNORECASE, + ) + # Server-blocked SQL patterns (save the round-trip by catching early) + _SQL_UNSUPPORTED_JOIN_RE = re.compile( + r"\b(?:CROSS\s+JOIN|RIGHT\s+(?:OUTER\s+)?JOIN|FULL\s+(?:OUTER\s+)?JOIN)\b", + re.IGNORECASE, + ) + _SQL_UNION_RE = re.compile(r"\bUNION\b", re.IGNORECASE) + _SQL_HAVING_RE = re.compile(r"\bHAVING\b", re.IGNORECASE) + _SQL_CTE_RE = re.compile(r"^\s*WITH\b", re.IGNORECASE) + _SQL_SUBQUERY_RE = re.compile( + r"\bIN\s*\(\s*SELECT\b|\bEXISTS\s*\(\s*SELECT\b|\(\s*SELECT\b.*\bFROM\b", + re.IGNORECASE, + ) + # SELECT * is intentionally rejected -- not a technical limitation but a + # deliberate design decision. Wide entities (e.g. account has 307 columns) + # make SELECT * extremely expensive on shared database infrastructure. + # COUNT(*) is NOT matched because COUNT appears before the *. + _SQL_SELECT_STAR_RE = re.compile( + r"\bSELECT\b\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?\*\s", + re.IGNORECASE, + ) + + def _sql_guardrails(self, sql: str) -> str: + """Apply safety guardrails to a SQL query before sending to the server. + + Checks split into two categories: + + **Blocked** (``ValidationError`` -- saves a server round-trip): + + 1. Write statements (INSERT/UPDATE/DELETE/DROP/etc.) + 2. CROSS JOIN, RIGHT JOIN, FULL OUTER JOIN (server rejects these) + 3. UNION / UNION ALL (server rejects) + 4. HAVING clause (server rejects) + 5. CTE / WITH clause (server rejects) + 6. Subqueries -- IN (SELECT ...), EXISTS (SELECT ...) (server rejects) + 7. SELECT * -- intentional design decision, not a technical limitation. + Wide entities make wildcard selects extremely expensive on shared + database infrastructure. ``COUNT(*)`` is not affected. + + **Warned** (``UserWarning`` -- query still executes): + + 8. Leading-wildcard LIKE (full table scan) + 9. Implicit cross join FROM a, b (cartesian product) + + All blocked patterns are also blocked by the server, but catching + them here saves the network round-trip and provides clearer error + messages. To bypass a specific check (e.g., if the server adds + support in the future), all checks are in this single method. + + :param sql: The SQL string (already stripped). + :return: The SQL string (unchanged). + :raises ValidationError: If the SQL contains a blocked pattern. + """ + # --- BLOCKED (save server round-trip) --- + + # 1. Block writes (strip SQL comments first to catch comment-prefixed writes) + sql_no_comments = self._SQL_COMMENT_RE.sub(" ", sql).strip() + if self._SQL_WRITE_RE.search(sql_no_comments): + raise ValidationError( + "SQL endpoint is read-only. Use client.records or " + "client.dataframe for write operations " + "(INSERT/UPDATE/DELETE are not supported).", + subcode=VALIDATION_SQL_WRITE_BLOCKED, + ) + + # 2. Block unsupported JOIN types + m = self._SQL_UNSUPPORTED_JOIN_RE.search(sql) + if m: + raise ValidationError( + f"Unsupported JOIN type: '{m.group(0).strip()}'. " + "Only INNER JOIN and LEFT JOIN are supported by the " + "Dataverse SQL endpoint.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 3. Block UNION + if self._SQL_UNION_RE.search(sql): + raise ValidationError( + "UNION is not supported by the Dataverse SQL endpoint. " + "Execute separate queries and combine results in Python " + "(e.g. pd.concat([df1, df2])).", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 4. Block HAVING + if self._SQL_HAVING_RE.search(sql): + raise ValidationError( + "HAVING is not supported by the Dataverse SQL endpoint. " + "Use WHERE to filter before GROUP BY instead.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 5. Block CTE / WITH + if self._SQL_CTE_RE.search(sql): + raise ValidationError( + "CTE (WITH ... AS) is not supported by the Dataverse SQL " + "endpoint. Use separate queries and combine in Python.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 6. Block subqueries + if self._SQL_SUBQUERY_RE.search(sql): + raise ValidationError( + "Subqueries are not supported by the Dataverse SQL " + "endpoint. Use separate SQL calls and combine results " + "in Python (e.g. step 1: get IDs, step 2: WHERE IN).", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # 7. Block SELECT * -- intentional design decision. + # Wide entities (e.g. account has 307 columns) make wildcard selects + # extremely expensive on shared database infrastructure. + # COUNT(*) is NOT matched: _SQL_SELECT_STAR_RE requires * to be the + # first token after SELECT/DISTINCT/TOP N, so COUNT appears before *. + if self._SQL_SELECT_STAR_RE.search(sql): + raise ValidationError( + "SELECT * is not supported. Specify column names explicitly " + "(e.g. SELECT name, revenue FROM account). " + "Use client.query.sql_columns('account') to discover available columns.", + subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX, + ) + + # --- WARNED (query still executes) --- + + # 8. Warn on leading-wildcard LIKE + if self._SQL_LEADING_WILDCARD_RE.search(sql): + warnings.warn( + "Query contains a leading-wildcard LIKE pattern " + "(e.g. LIKE '%value'). This forces a full table scan " + "and may degrade performance on large tables. " + "Prefer trailing wildcards (LIKE 'value%') when possible.", + UserWarning, + stacklevel=4, + ) + + # 9. Warn on implicit cross joins (server allows but risky) + if self._SQL_IMPLICIT_CROSS_JOIN_RE.search(sql): + warnings.warn( + "Query uses an implicit cross join (FROM table1, table2). " + "This produces a cartesian product that can generate " + "millions of intermediate rows and degrade shared database " + "performance. Use explicit JOIN...ON syntax instead: " + "FROM table1 a JOIN table2 b ON a.column = b.column", + UserWarning, + stacklevel=4, + ) + + return sql + + # ------------------------------------------------------------------ + # Lifecycle + # ------------------------------------------------------------------ + + def close(self) -> None: + """Clear in-memory caches and close the HTTP diagnostic logger. + + Called by subclass ``close()`` via ``super()``. Safe to call multiple times. + """ + self._logical_to_entityset_cache.clear() + self._logical_primaryid_cache.clear() + self._picklist_label_cache.clear() + if self._http_logger is not None: + self._http_logger.close() + self._http_logger = None + + # ------------------------------------------------------------------ + # Cache maintenance + # ------------------------------------------------------------------ + + def _flush_cache( + self, + kind, + ) -> int: + """Flush cached client metadata/state. + + :param kind: Cache kind to flush (only ``"picklist"`` supported). + :type kind: ``str`` + :return: Number of cache entries removed. + :rtype: ``int`` + :raises ValidationError: If ``kind`` is unsupported. + """ + k = (kind or "").strip().lower() + if k != "picklist": + raise ValidationError( + f"Unsupported cache kind '{kind}' (only 'picklist' is implemented)", + subcode=VALIDATION_UNSUPPORTED_CACHE_KIND, + ) + + removed = len(self._picklist_label_cache) + self._picklist_label_cache.clear() + return removed diff --git a/src/PowerPlatform/Dataverse/models/__init__.py b/src/PowerPlatform/Dataverse/models/__init__.py index dc10a4c0..50bd7326 100644 --- a/src/PowerPlatform/Dataverse/models/__init__.py +++ b/src/PowerPlatform/Dataverse/models/__init__.py @@ -8,12 +8,18 @@ - :class:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder`: Fluent query builder. - :mod:`~PowerPlatform.Dataverse.models.filters`: Composable OData filter expressions. +- :class:`~PowerPlatform.Dataverse.models.record.QueryResult`: Iterable result wrapper. - :class:`~PowerPlatform.Dataverse.models.upsert.UpsertItem`: Upsert operation item. Import directly from the specific module, e.g.:: from PowerPlatform.Dataverse.models.query_builder import QueryBuilder - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw + from PowerPlatform.Dataverse.models.record import QueryResult """ -__all__ = [] +from .filters import col, raw +from .protocol import DataverseModel +from .record import QueryResult + +__all__ = ["col", "raw", "DataverseModel", "QueryResult"] diff --git a/src/PowerPlatform/Dataverse/models/fetchxml_query.py b/src/PowerPlatform/Dataverse/models/fetchxml_query.py new file mode 100644 index 00000000..a1f43883 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/fetchxml_query.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""FetchXmlQuery — inert query object returned by QueryOperations.fetchxml().""" + +from __future__ import annotations + +import warnings +import xml.etree.ElementTree as _ET +from typing import Iterator, List, TYPE_CHECKING +from urllib.parse import unquote as _url_unquote, quote as _url_quote + +from ..core.errors import ValidationError +from .record import QueryResult, Record + +if TYPE_CHECKING: + from ..client import DataverseClient + + +__all__ = ["FetchXmlQuery"] + +_PREFER_HEADER = ( + "odata.include-annotations=" '"Microsoft.Dynamics.CRM.fetchxmlpagingcookie,' 'Microsoft.Dynamics.CRM.morerecords"' +) + +# Documented Dataverse GET request URL limit. See: +# learn.microsoft.com/power-apps/developer/data-platform/webapi/compose-http-requests-handle-errors#maximum-url-length +# FetchXML queries with many attributes or conditions are the most common way to reach it. +# $batch POST doubles this to 64 KB. +_MAX_URL_LENGTH = 32_768 +# Guards against infinite paging loops caused by a bug in cookie propagation or an +# unexpected server response. At the default Dataverse page size of 5,000 rows this +# cap allows up to 50 million records before raising; it is not a practical record +# limit but a circuit-breaker against runaway iteration. +_MAX_PAGES = 10_000 + + +class FetchXmlQuery: + """Inert FetchXML query object. No HTTP request is made until + :meth:`execute` or :meth:`execute_pages` is called. + + Obtained via ``client.query.fetchxml(xml)``. + + :param xml: Stripped, well-formed FetchXML string. + :param entity_name: Entity schema name from the ```` element. + :param client: Parent :class:`~PowerPlatform.Dataverse.client.DataverseClient`. + """ + + def __init__(self, xml: str, entity_name: str, client: "DataverseClient") -> None: + self._xml = xml + self._entity_name = entity_name + self._client = client + + def execute(self) -> QueryResult: + """Execute the FetchXML query and return all results as a :class:`QueryResult`. + + Blocking — fetches all pages upfront and holds every record in memory before + returning. Simple for small-to-medium result sets; use :meth:`execute_pages` + when the result set may be large or you want to process records as they arrive. + + :return: All matching records across all pages. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + rows = client.query.fetchxml(xml).execute() + df = rows.to_dataframe() + """ + all_records: List[Record] = [] + for page in self.execute_pages(): + all_records.extend(page.records) + return QueryResult(all_records) + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming — each iteration fires one HTTP request and yields one page. + Prefer over :meth:`execute` when: + + - The result set may be large and you do not want all records in memory at once. + - You want early exit: stop iterating once you find what you need and the + remaining HTTP round-trips are skipped automatically. + - You need per-page progress reporting or batched downstream writes. + + One-shot — do not iterate more than once. + + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.query.fetchxml(xml).execute_pages(): + process(page.to_dataframe()) + """ + current_xml = self._xml + page_num = 1 + page_count = 0 + + with self._client._scoped_odata() as od: + entity_set = od._entity_set_from_schema_name(self._entity_name) + base_url = f"{od.api}/{entity_set}" + + while True: + page_count += 1 + if page_count > _MAX_PAGES: + raise ValidationError( + f"FetchXML paging exceeded {_MAX_PAGES} pages. " + "This may indicate a runaway query or a bug in paging cookie propagation." + ) + + encoded_len = len(base_url) + len("?fetchXml=") + len(_url_quote(current_xml, safe="")) + if encoded_len > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML request URL exceeds {_MAX_URL_LENGTH} characters after encoding. " + "Simplify the query or reduce attributes/conditions." + ) + + r = od._request( + "get", + base_url, + headers={"Prefer": _PREFER_HEADER}, + params={"fetchXml": current_xml}, + ) + data = r.json() if hasattr(r, "json") else {} + items = data.get("value") if isinstance(data, dict) else None + page_records: List[Record] = [] + if isinstance(items, list): + for item in items: + if isinstance(item, dict): + page_records.append(Record.from_api_response(self._entity_name, item)) + + yield QueryResult(page_records) + + more_raw = data.get("@Microsoft.Dynamics.CRM.morerecords", False) if isinstance(data, dict) else False + more = more_raw is True or (isinstance(more_raw, str) and more_raw.lower() == "true") + if not more: + break + + raw_cookie = ( + data.get("@Microsoft.Dynamics.CRM.fetchxmlpagingcookie", "") if isinstance(data, dict) else "" + ) + + _cookie_parse_error = False + if raw_cookie: + try: + cookie_el = _ET.fromstring(raw_cookie) + inner_encoded = cookie_el.get("pagingcookie", "") + if inner_encoded: + cookie = _url_unquote(_url_unquote(inner_encoded)) + page_num = int(cookie_el.get("pagenumber", str(page_num + 1))) + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("paging-cookie", cookie) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") + continue + except (_ET.ParseError, ValueError) as exc: + warnings.warn( + f"FetchXML paging cookie could not be parsed ({exc}); " "falling back to simple paging.", + UserWarning, + stacklevel=2, + ) + _cookie_parse_error = True + + # Simple paging fallback: server returned morerecords=true but no paging + # cookie. Dataverse omits the cookie when the query cannot use cookie-based + # paging (e.g. FetchXML ordered by a link-entity column). We continue with + # page-number-only paging rather than truncating, but warn because simple + # paging has a 50,000-record server cap and performance degrades at high page + # numbers. The caller may be able to avoid this by reordering on the root + # entity instead. + if not _cookie_parse_error: + warnings.warn( + "Dataverse did not return a paging cookie; falling back to simple paging " + "(page-number increment only). Simple paging is capped at 50,000 records " + "and degrades in performance at high page numbers. Consider reordering on " + "a root-entity column to enable cookie-based paging.", + UserWarning, + stacklevel=2, + ) + page_num += 1 + fetch_el = _ET.fromstring(current_xml) + fetch_el.set("page", str(page_num)) + current_xml = _ET.tostring(fetch_el, encoding="unicode") diff --git a/src/PowerPlatform/Dataverse/models/filters.py b/src/PowerPlatform/Dataverse/models/filters.py index c5de258c..2a7929cc 100644 --- a/src/PowerPlatform/Dataverse/models/filters.py +++ b/src/PowerPlatform/Dataverse/models/filters.py @@ -10,24 +10,26 @@ Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt, filter_in + from PowerPlatform.Dataverse.models.filters import col, raw - # Simple comparison - expr = eq("statecode", 0) + # Preferred GA idiom — col() proxy + expr = col("statecode") == 0 print(expr.to_odata()) # statecode eq 0 # Complex composition with OR and AND - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = (col("statecode") == 0) | (col("statecode") == 1) & (col("revenue") > 100000) print(expr.to_odata()) - # ((statecode eq 0 or statecode eq 1) and revenue gt 100000) - # In operator (Dataverse function) - expr = filter_in("statecode", [0, 1, 2]) + # In / not-in + expr = col("statecode").in_([0, 1, 2]) print(expr.to_odata()) # Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"]) + # Raw OData escape hatch (no deprecation warning) + expr = raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + # Negation - expr = ~eq("statecode", 1) + expr = ~(col("statecode") == 1) print(expr.to_odata()) # not (statecode eq 1) """ @@ -35,11 +37,16 @@ import enum import uuid +import warnings from datetime import date, datetime, timezone -from typing import Any, Collection, Sequence +from typing import Any, Collection, List __all__ = [ "FilterExpression", + "ColumnProxy", + "col", + "raw", + # Deprecated factories — still functional, fire DeprecationWarning on call: "eq", "ne", "gt", @@ -55,7 +62,6 @@ "filter_in", "not_in", "not_between", - "raw", ] @@ -264,138 +270,458 @@ def to_odata(self) -> str: # --------------------------------------------------------------------------- -# Public factory functions +# Private implementation helpers (no warnings — used internally and by col()) # --------------------------------------------------------------------------- -def eq(column: str, value: Any) -> FilterExpression: - """Equality filter: ``column eq value``. +def _eq_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "eq", value) + + +def _ne_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ne", value) + + +def _gt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "gt", value) + + +def _ge_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "ge", value) + + +def _lt_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "lt", value) + + +def _le_impl(column: str, value: Any) -> FilterExpression: + return _ComparisonFilter(column, "le", value) + + +def _contains_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("contains", column, value) + + +def _startswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("startswith", column, value) + + +def _endswith_impl(column: str, value: str) -> FilterExpression: + return _FunctionFilter("endswith", column, value) + + +def _in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _InFilter(column, values) + + +def _not_in_impl(column: str, values: Collection[Any]) -> FilterExpression: + return _NotInFilter(column, values) + + +# --------------------------------------------------------------------------- +# ColumnProxy — GA idiom for building filter expressions +# --------------------------------------------------------------------------- - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. +_LIKE_WILDCARD = "%" + + +def _compile_like(column: str, pattern: str) -> FilterExpression: + """Compile a LIKE-style pattern to an OData FilterExpression. + + Pattern rules: + - ``val%`` → ``startswith(column, 'val')`` + - ``%val`` → ``endswith(column, 'val')`` + - ``%val%`` → ``contains(column, 'val')`` + - ``val`` (no wildcard) → ``column eq 'val'`` (equality) + - Anything else → :class:`ValueError` + + :param column: Lowercased column name. + :param pattern: The LIKE pattern string. + :raises ValueError: If the pattern contains wildcards in unsupported positions. + """ + has_start = pattern.startswith(_LIKE_WILDCARD) + has_end = pattern.endswith(_LIKE_WILDCARD) + inner = pattern.strip(_LIKE_WILDCARD) + + # Detect non-reducible interior wildcards: after stripping the leading/trailing + # % the inner value must contain no further % characters. + if _LIKE_WILDCARD in inner: + raise ValueError( + f"like() pattern {pattern!r} is not reducible to a single OData function. " + "Use raw(), fetchxml(), or query.sql() for complex wildcard patterns." + ) + + if not has_start and has_end: + # "val%" — startswith + return _startswith_impl(column, inner) + if has_start and not has_end: + # "%val" — endswith + return _endswith_impl(column, inner) + if has_start and has_end: + # "%val%" — contains + return _contains_impl(column, inner) + # No wildcard at all — exact equality + return _eq_impl(column, pattern) + + +class ColumnProxy: + """Fluent proxy for building OData filter expressions from a column name. + + Returned by :func:`col`. Operator overloads and methods produce + :class:`FilterExpression` instances that can be passed to + ``QueryBuilder.where()``. Example:: - eq("statecode", 0).to_odata() # "statecode eq 0" + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 # equality + expr = col("revenue") > 1_000_000 # comparison + expr = col("name").like("Contoso%") # startswith + expr = col("name").is_null() # null check + expr = col("statecode").in_([0, 1]) # in """ - return _ComparisonFilter(column, "eq", value) + + __slots__ = ("_column",) + + def __init__(self, name: str) -> None: + if not name or not name.strip(): + raise ValueError("col() requires a non-empty column name") + self._column = name.strip().lower() + + # ---------------------------------------------------------------- comparisons + + def __eq__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _eq_impl(self._column, other) + + def __ne__(self, other: Any) -> FilterExpression: # type: ignore[override] + return _ne_impl(self._column, other) + + def __gt__(self, other: Any) -> FilterExpression: + return _gt_impl(self._column, other) + + def __ge__(self, other: Any) -> FilterExpression: + return _ge_impl(self._column, other) + + def __lt__(self, other: Any) -> FilterExpression: + return _lt_impl(self._column, other) + + def __le__(self, other: Any) -> FilterExpression: + return _le_impl(self._column, other) + + # ---------------------------------------------------------------- null checks + + def is_null(self) -> FilterExpression: + """Column equals null: ``column eq null``.""" + return _eq_impl(self._column, None) + + def is_not_null(self) -> FilterExpression: + """Column not null: ``column ne null``.""" + return _ne_impl(self._column, None) + + # ---------------------------------------------------------------- in / not-in + + def in_(self, values: Collection[Any]) -> FilterExpression: + """In filter using ``Microsoft.Dynamics.CRM.In``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _in_impl(self._column, values) + + def not_in(self, values: Collection[Any]) -> FilterExpression: + """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. + + :param values: Non-empty collection of values. + :raises ValueError: If ``values`` is empty. + """ + return _not_in_impl(self._column, values) + + # ---------------------------------------------------------------- range + + def between(self, lo: Any, hi: Any) -> FilterExpression: + """Between filter: ``(column ge lo and column le hi)``.""" + return _ge_impl(self._column, lo) & _le_impl(self._column, hi) + + def not_between(self, lo: Any, hi: Any) -> FilterExpression: + """Not-between filter: ``not (column ge lo and column le hi)``.""" + return ~(self.between(lo, hi)) + + # ---------------------------------------------------------------- string functions + + def contains(self, value: str) -> FilterExpression: + """Contains filter: ``contains(column, value)``.""" + return _contains_impl(self._column, value) + + def startswith(self, value: str) -> FilterExpression: + """Startswith filter: ``startswith(column, value)``.""" + return _startswith_impl(self._column, value) + + def endswith(self, value: str) -> FilterExpression: + """Endswith filter: ``endswith(column, value)``.""" + return _endswith_impl(self._column, value) + + # ---------------------------------------------------------------- like / not_like + + def like(self, pattern: str) -> FilterExpression: + """Pattern-match filter compiled to the closest OData equivalent. + + +-----------------+-----------------------------+-------------------------------------+ + | Pattern form | Example | Compiles to | + +=================+=============================+=====================================+ + | ``val%`` | ``like("Contoso%")`` | ``startswith(column,'Contoso')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val`` | ``like("%Ltd")`` | ``endswith(column,'Ltd')`` | + +-----------------+-----------------------------+-------------------------------------+ + | ``%val%`` | ``like("%Corp%")`` | ``contains(column,'Corp')`` | + +-----------------+-----------------------------+-------------------------------------+ + | No wildcard | ``like("Contoso")`` | ``column eq 'Contoso'`` | + +-----------------+-----------------------------+-------------------------------------+ + | Other | ``like("Con%oso")`` | :class:`ValueError` | + +-----------------+-----------------------------+-------------------------------------+ + + :param pattern: LIKE-style pattern string. + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return _compile_like(self._column, pattern) + + def not_like(self, pattern: str) -> FilterExpression: + """Negated pattern-match filter; mirrors :meth:`like` rules then negates. + + :param pattern: LIKE-style pattern string (same rules as :meth:`like`). + :raises ValueError: If the pattern cannot be reduced to a single OData function. + """ + return ~_compile_like(self._column, pattern) + + # ---------------------------------------------------------------- hash / repr + + def __hash__(self) -> int: + return hash(self._column) + + def __repr__(self) -> str: + return f"ColumnProxy({self._column!r})" + + +# --------------------------------------------------------------------------- +# Public factory: col() — no deprecation warning +# --------------------------------------------------------------------------- + + +def col(name: str) -> ColumnProxy: + """Return a :class:`ColumnProxy` for building filter expressions. + + This is the preferred GA idiom for constructing filter expressions:: + + from PowerPlatform.Dataverse.models.filters import col + + expr = col("statecode") == 0 + expr = col("revenue") > 1_000_000 + expr = col("name").like("Contoso%") + expr = col("statecode").in_([0, 1]) + expr = col("parentaccountid").is_null() + + :param name: Column logical name (case-insensitive, will be lowercased). + :return: A :class:`ColumnProxy` bound to the column. + :raises ValueError: If ``name`` is empty. + """ + return ColumnProxy(name) + + +# --------------------------------------------------------------------------- +# Public factory: raw() — no deprecation warning (OData escape hatch) +# --------------------------------------------------------------------------- + + +def raw(filter_string: str) -> FilterExpression: + """Verbatim OData filter expression (passed through unchanged). + + This function is **not** deprecated — it is the OData escape hatch with + no typed replacement. + + :param filter_string: Raw OData filter string. + :return: A :class:`FilterExpression`. + + Example:: + + raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") + """ + return _RawFilter(filter_string) + + +# --------------------------------------------------------------------------- +# Deprecated public factory functions — fire DeprecationWarning on CALL +# --------------------------------------------------------------------------- + +_DEP_MSG = "'{name}' is deprecated and will be removed in a future release. " "Use {replacement} instead." + + +def eq(column: str, value: Any) -> FilterExpression: + """Equality filter: ``column eq value``. + + .. deprecated:: + Use ``col(column) == value`` instead. + """ + warnings.warn( + _DEP_MSG.format(name="eq", replacement="col('column') == value"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, value) def ne(column: str, value: Any) -> FilterExpression: """Not-equal filter: ``column ne value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) != value`` instead. """ - return _ComparisonFilter(column, "ne", value) + warnings.warn( + _DEP_MSG.format(name="ne", replacement="col('column') != value"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, value) def gt(column: str, value: Any) -> FilterExpression: """Greater-than filter: ``column gt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) > value`` instead. """ - return _ComparisonFilter(column, "gt", value) + warnings.warn( + _DEP_MSG.format(name="gt", replacement="col('column') > value"), + DeprecationWarning, + stacklevel=2, + ) + return _gt_impl(column, value) def ge(column: str, value: Any) -> FilterExpression: """Greater-than-or-equal filter: ``column ge value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) >= value`` instead. """ - return _ComparisonFilter(column, "ge", value) + warnings.warn( + _DEP_MSG.format(name="ge", replacement="col('column') >= value"), + DeprecationWarning, + stacklevel=2, + ) + return _ge_impl(column, value) def lt(column: str, value: Any) -> FilterExpression: """Less-than filter: ``column lt value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) < value`` instead. """ - return _ComparisonFilter(column, "lt", value) + warnings.warn( + _DEP_MSG.format(name="lt", replacement="col('column') < value"), + DeprecationWarning, + stacklevel=2, + ) + return _lt_impl(column, value) def le(column: str, value: Any) -> FilterExpression: """Less-than-or-equal filter: ``column le value``. - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: A filter expression. + .. deprecated:: + Use ``col(column) <= value`` instead. """ - return _ComparisonFilter(column, "le", value) + warnings.warn( + _DEP_MSG.format(name="le", replacement="col('column') <= value"), + DeprecationWarning, + stacklevel=2, + ) + return _le_impl(column, value) def contains(column: str, value: str) -> FilterExpression: """Contains filter: ``contains(column, value)``. - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: A filter expression. + .. deprecated:: + Use ``col(column).contains(value)`` instead. """ - return _FunctionFilter("contains", column, value) + warnings.warn( + _DEP_MSG.format(name="contains", replacement="col('column').contains(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _contains_impl(column, value) def startswith(column: str, value: str) -> FilterExpression: """Startswith filter: ``startswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).startswith(value)`` instead. """ - return _FunctionFilter("startswith", column, value) + warnings.warn( + _DEP_MSG.format(name="startswith", replacement="col('column').startswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _startswith_impl(column, value) def endswith(column: str, value: str) -> FilterExpression: """Endswith filter: ``endswith(column, value)``. - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: A filter expression. + .. deprecated:: + Use ``col(column).endswith(value)`` instead. """ - return _FunctionFilter("endswith", column, value) + warnings.warn( + _DEP_MSG.format(name="endswith", replacement="col('column').endswith(value)"), + DeprecationWarning, + stacklevel=2, + ) + return _endswith_impl(column, value) def between(column: str, low: Any, high: Any) -> FilterExpression: """Between filter: ``(column ge low and column le high)``. - Syntactic sugar that composes :func:`ge` and :func:`le` with ``&``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: A composed filter expression. - - Example:: - - between("revenue", 100000, 500000).to_odata() - # "(revenue ge 100000 and revenue le 500000)" + .. deprecated:: + Use ``col(column).between(low, high)`` instead. """ - return ge(column, low) & le(column, high) + warnings.warn( + _DEP_MSG.format(name="between", replacement="col('column').between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through the deprecated ge/le wrappers + return _ge_impl(column, low) & _le_impl(column, high) def is_null(column: str) -> FilterExpression: """Null check: ``column eq null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_null()`` instead. """ - return _ComparisonFilter(column, "eq", None) + warnings.warn( + _DEP_MSG.format(name="is_null", replacement="col('column').is_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _eq_impl(column, None) def is_not_null(column: str) -> FilterExpression: """Not-null check: ``column ne null``. - :param column: Column name (will be lowercased). - :return: A filter expression. + .. deprecated:: + Use ``col(column).is_not_null()`` instead. """ - return _ComparisonFilter(column, "ne", None) + warnings.warn( + _DEP_MSG.format(name="is_not_null", replacement="col('column').is_not_null()"), + DeprecationWarning, + stacklevel=2, + ) + return _ne_impl(column, None) def filter_in(column: str, values: Collection[Any]) -> FilterExpression: @@ -403,63 +729,45 @@ def filter_in(column: str, values: Collection[Any]) -> FilterExpression: Named ``filter_in`` because ``in`` is a Python keyword. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. - :raises ValueError: If ``values`` is empty. - - Example:: + .. deprecated:: + Use ``col(column).in_(values)`` instead. - filter_in("statecode", [0, 1, 2]).to_odata() - # "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=["0","1","2"])" + :raises ValueError: If ``values`` is empty. """ - return _InFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="filter_in", replacement="col('column').in_(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _in_impl(column, values) def not_in(column: str, values: Collection[Any]) -> FilterExpression: """Not-in filter using ``Microsoft.Dynamics.CRM.NotIn``. - Named ``not_in`` to parallel :func:`filter_in`. + .. deprecated:: + Use ``col(column).not_in(values)`` instead. - :param column: Column name (will be lowercased). - :param values: Non-empty sequence of values. - :return: A filter expression. :raises ValueError: If ``values`` is empty. - - Example:: - - not_in("statecode", [0, 1]).to_odata() - # "Microsoft.Dynamics.CRM.NotIn(PropertyName='statecode',PropertyValues=[\"0\",\"1\"])" """ - return _NotInFilter(column, values) + warnings.warn( + _DEP_MSG.format(name="not_in", replacement="col('column').not_in(values)"), + DeprecationWarning, + stacklevel=2, + ) + return _not_in_impl(column, values) def not_between(column: str, low: Any, high: Any) -> FilterExpression: """Not-between filter: ``not (column ge low and column le high)``. - Syntactic sugar that negates :func:`between` with ``~``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: A composed filter expression. - - Example:: - - not_between("revenue", 100000, 500000).to_odata() - # "not ((revenue ge 100000 and revenue le 500000))" + .. deprecated:: + Use ``col(column).not_between(low, high)`` instead. """ - return ~between(column, low, high) - - -def raw(filter_string: str) -> FilterExpression: - """Verbatim OData filter expression (passed through unchanged). - - :param filter_string: Raw OData filter string. - :return: A filter expression. - - Example:: - - raw("Microsoft.Dynamics.CRM.Today(PropertyName='createdon')") - """ - return _RawFilter(filter_string) + warnings.warn( + _DEP_MSG.format(name="not_between", replacement="col('column').not_between(low, high)"), + DeprecationWarning, + stacklevel=2, + ) + # Use private helpers to avoid chaining through deprecated ge/le wrappers + return ~(_ge_impl(column, low) & _le_impl(column, high)) diff --git a/src/PowerPlatform/Dataverse/models/protocol.py b/src/PowerPlatform/Dataverse/models/protocol.py new file mode 100644 index 00000000..d9b0e502 --- /dev/null +++ b/src/PowerPlatform/Dataverse/models/protocol.py @@ -0,0 +1,77 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""DataverseModel structural Protocol for typed entity integration.""" + +from __future__ import annotations + +from typing import Protocol, runtime_checkable + +__all__ = ["DataverseModel"] + + +@runtime_checkable +class DataverseModel(Protocol): + """Structural Protocol enabling typed entity instances to be passed to + ``records.create()`` and ``records.update()``. + + Implement this Protocol on any entity class (dataclass, Pydantic model, + hand-rolled) to enable it to be passed directly to CRUD operations without + specifying the table name or converting to dict manually. + + Required class variables: + + - ``__entity_logical_name__`` — Dataverse logical entity name (e.g. ``"account"``) + - ``__entity_set_name__`` — OData entity set name (e.g. ``"accounts"``) + + Required instance methods: + + - ``to_dict()`` — return record payload as ``dict`` + - ``from_dict(data)`` — classmethod to reconstruct from a response ``dict`` + + Example:: + + from dataclasses import dataclass + from PowerPlatform.Dataverse import DataverseModel + + @dataclass + class Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "Account": + return cls( + name=data.get("name", ""), + telephone1=data.get("telephone1", ""), + ) + + # isinstance() works today — Protocol is runtime_checkable: + assert isinstance(Account(), DataverseModel) + + # Type your own helpers against the Protocol now: + def save(entity: DataverseModel) -> None: + data = entity.to_dict() + client.records.create(entity.__entity_logical_name__, data) + + Note: + Direct dispatch (``client.records.create(entity)`` without a table name + or dict) is not yet supported and will be added in a future release. + """ + + __entity_logical_name__: str + __entity_set_name__: str + + def to_dict(self) -> dict: + """Return the record payload as a plain dictionary.""" + ... + + @classmethod + def from_dict(cls, data: dict) -> DataverseModel: + """Reconstruct an instance from a response dictionary.""" + ... diff --git a/src/PowerPlatform/Dataverse/models/query_builder.py b/src/PowerPlatform/Dataverse/models/query_builder.py index dbd79b36..bb2664fe 100644 --- a/src/PowerPlatform/Dataverse/models/query_builder.py +++ b/src/PowerPlatform/Dataverse/models/query_builder.py @@ -10,57 +10,74 @@ Example:: # Via client (recommended) -- flat iteration over records + from PowerPlatform.Dataverse.models.filters import col + for record in (client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1_000_000) .order_by("revenue", descending=True) .top(100) .execute()): print(record["name"]) # With composable expression tree - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col, raw for record in (client.query.builder("account") .select("name", "revenue") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000)) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) .top(100) .execute()): print(record["name"]) - # Opt-in paged iteration (for batch processing) + # Lazy paged iteration (one QueryResult per HTTP page) for page in (client.query.builder("account") .select("name") - .execute(by_page=True)): + .execute_pages()): process_batch(page) # Get results as a pandas DataFrame df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ from __future__ import annotations -from typing import Any, Collection, Dict, Iterable, List, Optional, Sequence, TypedDict, Union +import sys +import warnings +from typing import Any, Iterator, List, Optional, TypedDict, Union + +# typing.Self (PEP 673, Python 3.11+) makes fluent methods return the concrete +# subclass type. TypeVar fallback for Python 3.10 uses the same name so docs render identically. +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing import TypeVar + + Self = TypeVar("Self", bound="_QueryBuilderBase") # type: ignore[assignment] import pandas as pd from . import filters -from .record import Record +from .record import QueryResult, Record __all__ = ["QueryBuilder", "QueryParams", "ExpandOption"] +# Sentinel for detecting when by_page is explicitly passed to execute() +_BY_PAGE_UNSET = object() + class QueryParams(TypedDict, total=False): """Typed dictionary returned by :meth:`QueryBuilder.build`. Provides IDE autocomplete when passing build results to - ``client.records.get()`` manually. + ``client.records.list()`` manually. """ table: str @@ -164,27 +181,17 @@ def to_odata(self) -> str: return self.relation -class QueryBuilder: - """Fluent interface for building OData queries. - - Provides method chaining for constructing complex queries with - type-safe filter operations. Can be used standalone (via :meth:`build`) - or bound to a client (via :meth:`execute`). - - :param table: Table schema name to query. - :type table: str - :raises ValueError: If ``table`` is empty. +class _QueryBuilderBase: + """Pure fluent interface for building OData queries — no I/O. - Example: - Standalone query construction:: + Holds all query state and chaining methods (``select``, ``where``, + ``order_by``, ``top``, ``page_size``, ``count``, ``expand``, + ``include_annotations``, ``include_formatted_values``) and + :meth:`build`. - query = (QueryBuilder("account") - .select("name") - .filter_eq("statecode", 0) - .top(10)) - params = query.build() - # {"table": "account", "select": ["name"], - # "filter": "statecode eq 0", "top": 10} + Subclasses add execution: :class:`QueryBuilder` for sync clients, + :class:`~PowerPlatform.Dataverse.aio.models.async_query_builder.AsyncQueryBuilder` + for async clients. """ def __init__(self, table: str) -> None: @@ -204,7 +211,7 @@ def __init__(self, table: str) -> None: # ----------------------------------------------------------------- select - def select(self, *columns: str) -> QueryBuilder: + def select(self, *columns: str) -> Self: """Select specific columns to retrieve. Column names are passed as-is; the OData layer lowercases them @@ -220,222 +227,16 @@ def select(self, *columns: str) -> QueryBuilder: self._select.extend(columns) return self - # ----------------------------------------------------------- filter: comparison - - def filter_eq(self, column: str, value: Any) -> QueryBuilder: - """Add equality filter: ``column eq value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.eq(column, value)) - return self - - def filter_ne(self, column: str, value: Any) -> QueryBuilder: - """Add not-equal filter: ``column ne value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ne(column, value)) - return self - - def filter_gt(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than filter: ``column gt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.gt(column, value)) - return self - - def filter_ge(self, column: str, value: Any) -> QueryBuilder: - """Add greater-than-or-equal filter: ``column ge value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.ge(column, value)) - return self - - def filter_lt(self, column: str, value: Any) -> QueryBuilder: - """Add less-than filter: ``column lt value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.lt(column, value)) - return self - - def filter_le(self, column: str, value: Any) -> QueryBuilder: - """Add less-than-or-equal filter: ``column le value``. - - :param column: Column name (will be lowercased). - :param value: Value to compare against. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.le(column, value)) - return self - - # --------------------------------------------------------- filter: string functions - - def filter_contains(self, column: str, value: str) -> QueryBuilder: - """Add contains filter: ``contains(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Substring to search for. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.contains(column, value)) - return self - - def filter_startswith(self, column: str, value: str) -> QueryBuilder: - """Add startswith filter: ``startswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Prefix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.startswith(column, value)) - return self - - def filter_endswith(self, column: str, value: str) -> QueryBuilder: - """Add endswith filter: ``endswith(column, value)``. - - :param column: Column name (will be lowercased). - :param value: Suffix to match. - :return: Self for method chaining. - """ - self._filter_parts.append(filters.endswith(column, value)) - return self - - # --------------------------------------------------------- filter: null checks - - def filter_null(self, column: str) -> QueryBuilder: - """Add null check: ``column eq null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_null(column)) - return self - - def filter_not_null(self, column: str) -> QueryBuilder: - """Add not-null check: ``column ne null``. - - :param column: Column name (will be lowercased). - :return: Self for method chaining. - """ - self._filter_parts.append(filters.is_not_null(column)) - return self - - # --------------------------------------------------------- filter: special - - def filter_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add an ``in`` filter using ``Microsoft.Dynamics.CRM.In``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values for the ``in`` clause. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - # Produces: Microsoft.Dynamics.CRM.In( - # PropertyName='statecode',PropertyValues=["0","1","2"]) - """ - self._filter_parts.append(filters.filter_in(column, values)) - return self - - def filter_not_in(self, column: str, values: Collection[Any]) -> QueryBuilder: - """Add a ``not in`` filter using ``Microsoft.Dynamics.CRM.NotIn``. - - :param column: Column name (will be lowercased). - :param values: Non-empty list of values to exclude. - :return: Self for method chaining. - :raises ValueError: If ``values`` is empty. - - Example:: - - query = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - # Produces: Microsoft.Dynamics.CRM.NotIn( - # PropertyName='statecode',PropertyValues=["2","3"]) - """ - self._filter_parts.append(filters.not_in(column, values)) - return self - - def filter_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a between filter: ``(column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive). - :param high: Upper bound (inclusive). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_between("revenue", 100000, 500000) - # Produces: (revenue ge 100000 and revenue le 500000) - """ - self._filter_parts.append(filters.between(column, low, high)) - return self - - def filter_not_between(self, column: str, low: Any, high: Any) -> QueryBuilder: - """Add a not-between filter: ``not (column ge low and column le high)``. - - :param column: Column name (will be lowercased). - :param low: Lower bound (inclusive, will be excluded). - :param high: Upper bound (inclusive, will be excluded). - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - # Produces: not ((revenue ge 100000 and revenue le 500000)) - """ - self._filter_parts.append(filters.not_between(column, low, high)) - return self - - def filter_raw(self, filter_string: str) -> QueryBuilder: - """Add a raw OData filter string. - - Use this for complex filters not covered by other methods. - Column names in the filter string should be lowercase. - - .. warning:: - The filter string is passed directly to Dataverse without validation. - Ensure it follows OData filter syntax; a malformed expression will result - in a ``400 Bad Request`` error from the server. - - :param filter_string: Raw OData filter expression. - :return: Self for method chaining. - - Example:: - - query = QueryBuilder("account").filter_raw( - "(statecode eq 0 or statecode eq 1)" - ) - """ - self._filter_parts.append(filters.raw(filter_string)) - return self - # ------------------------------------------------------ filter: expression tree - def where(self, expression: filters.FilterExpression) -> QueryBuilder: + def where(self, expression: filters.FilterExpression) -> Self: """Add a composable filter expression. Accepts a :class:`~PowerPlatform.Dataverse.models.filters.FilterExpression` - built using the convenience functions from - :mod:`~PowerPlatform.Dataverse.models.filters`. + built using :func:`~PowerPlatform.Dataverse.models.filters.col` or + :func:`~PowerPlatform.Dataverse.models.filters.raw`. - Multiple ``where()`` calls and ``filter_*()`` calls are all - AND-joined together in the order they were called. + Multiple ``where()`` calls are AND-joined together in call order. :param expression: A composable filter expression. :type expression: FilterExpression @@ -444,11 +245,11 @@ def where(self, expression: filters.FilterExpression) -> QueryBuilder: Example:: - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col query = (QueryBuilder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) - & gt("revenue", 100000))) + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000)) """ if not isinstance(expression, filters.FilterExpression): raise TypeError(f"where() requires a FilterExpression, got {type(expression).__name__}") @@ -457,7 +258,7 @@ def where(self, expression: filters.FilterExpression) -> QueryBuilder: # --------------------------------------------------------------- ordering - def order_by(self, column: str, descending: bool = False) -> QueryBuilder: + def order_by(self, column: str, descending: bool = False) -> Self: """Add sorting order. Can be called multiple times for multi-column sorting. @@ -472,7 +273,7 @@ def order_by(self, column: str, descending: bool = False) -> QueryBuilder: # --------------------------------------------------------------- pagination - def top(self, count: int) -> QueryBuilder: + def top(self, count: int) -> Self: """Limit the total number of results. :param count: Maximum number of records to return (must be >= 1). @@ -484,7 +285,7 @@ def top(self, count: int) -> QueryBuilder: self._top = count return self - def page_size(self, size: int) -> QueryBuilder: + def page_size(self, size: int) -> Self: """Set the number of records per page. Controls how many records are returned in each page/batch @@ -499,7 +300,7 @@ def page_size(self, size: int) -> QueryBuilder: self._page_size = size return self - def count(self) -> QueryBuilder: + def count(self) -> Self: """Request a count of matching records in the response. Adds ``$count=true`` to the query, causing the server to include @@ -511,14 +312,14 @@ def count(self) -> QueryBuilder: Example:: results = (client.query.builder("account") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .count() .execute()) """ self._count = True return self - def include_formatted_values(self) -> QueryBuilder: + def include_formatted_values(self) -> Self: """Request formatted values in the response. Adds ``Prefer: odata.include-annotations="OData.Community.Display.V1.FormattedValue"`` @@ -548,7 +349,7 @@ def include_formatted_values(self) -> QueryBuilder: self._include_annotations = "OData.Community.Display.V1.FormattedValue" return self - def include_annotations(self, annotation: str = "*") -> QueryBuilder: + def include_annotations(self, annotation: str = "*") -> Self: """Request specific OData annotations in the response. Sets the ``Prefer: odata.include-annotations`` header. Use ``"*"`` @@ -576,7 +377,7 @@ def include_annotations(self, annotation: str = "*") -> QueryBuilder: # --------------------------------------------------------------- expand - def expand(self, *relations: Union[str, ExpandOption]) -> QueryBuilder: + def expand(self, *relations: Union[str, ExpandOption]) -> Self: """Expand navigation properties. Accepts plain navigation property names (case-sensitive, passed @@ -612,8 +413,8 @@ def build(self) -> QueryParams: """Build query parameters dictionary. Returns a :class:`QueryParams` dictionary suitable for passing to - the OData layer. All ``filter_*()`` and ``where()`` clauses are - AND-joined into a single ``filter`` string in call order. + the OData layer. All ``where()`` clauses are AND-joined into a + single ``filter`` string in call order. :return: Dictionary with ``table`` and optionally ``select``, ``filter``, ``orderby``, ``expand``, ``top``, ``page_size``, @@ -645,148 +446,233 @@ def build(self) -> QueryParams: params["include_annotations"] = self._include_annotations return params - # --------------------------------------------------------------- guards - def _validate_constraints(self) -> None: - """Raise if the query has no limiting constraints. +class QueryBuilder(_QueryBuilderBase): + """Fluent interface for building and executing OData queries against a sync client. - At least one of ``select``, ``filter``, or ``top`` must be set - before executing a query to prevent accidental full-table scans. + Provides method chaining for constructing complex queries with + composable filter expressions. Can be used standalone (via :meth:`build`) + or bound to a client (via :meth:`execute`). - :raises ValueError: If none of ``select()``, ``filter_*()``, - ``where()``, or ``top()`` has been called. - """ - if not (self._select or self._filter_parts or self._top is not None): - raise ValueError( - "Unbounded query: set at least one of select(), filter_*(), " - "where(), or top() before calling execute() or to_dataframe()." - ) + :param table: Table schema name to query. + :type table: str + :raises ValueError: If ``table`` is empty. + + Example: + Standalone query construction:: + + from PowerPlatform.Dataverse.models.filters import col + + query = (QueryBuilder("account") + .select("name") + .where(col("statecode") == 0) + .top(10)) + params = query.build() + # {"table": "account", "select": ["name"], + # "filter": "statecode eq 0", "top": 10} + """ # --------------------------------------------------------------- execute - def execute(self, *, by_page: bool = False) -> Union[Iterable[Record], Iterable[List[Record]]]: + def execute(self, *, by_page=_BY_PAGE_UNSET) -> Union[QueryResult, Iterator[QueryResult]]: """Execute the query and return results. - By default, returns a flat iterator over individual records, - abstracting away OData paging. Pass ``by_page=True`` to get - page-level iteration instead (useful for batch processing). + Returns a :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected. Use :meth:`execute_pages` for lazy per-page + iteration. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. Standalone ``QueryBuilder`` instances should use :meth:`build` to get parameters and pass them - to ``client.records.get()`` manually. - - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``execute()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. - - :param by_page: If ``True``, yield pages (lists of - :class:`~PowerPlatform.Dataverse.models.record.Record` objects) - instead of individual records. Defaults to ``False``. - :type by_page: bool - :return: Generator yielding individual - :class:`~PowerPlatform.Dataverse.models.record.Record` objects - (default) or pages of records (when ``by_page=True``). - :rtype: Iterable[Record] or Iterable[List[Record]] - :raises ValueError: If no ``select``, ``filter``, or ``top`` + to ``client.records.list()`` manually. + + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. + + .. deprecated:: + The ``by_page`` parameter is deprecated. Use :meth:`execute_pages` + for lazy per-page iteration, or plain ``execute()`` (no flag) for + the default eager result. + + :return: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + with all pages collected (default), or page iterator (deprecated + ``by_page=True``). + :rtype: QueryResult or Iterator[QueryResult] + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. - Example: - Flat iteration (default):: + Example:: - for record in (client.query.builder("account") - .select("name") - .filter_eq("statecode", 0) - .execute()): - print(record["name"]) + from PowerPlatform.Dataverse.models.filters import col - Paged iteration:: + for record in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute()): + print(record["name"]) + """ + use_by_page = False + if by_page is not _BY_PAGE_UNSET: + use_by_page = bool(by_page) + if use_by_page: + warnings.warn( + "'execute(by_page=True)' is deprecated; use 'execute_pages()' instead.", + UserWarning, + stacklevel=2, + ) + else: + warnings.warn( + "'execute(by_page=False)' is deprecated; " + "the by_page flag is redundant — use plain 'execute()' instead.", + UserWarning, + stacklevel=2, + ) - for page in (client.query.builder("account") - .select("name") - .execute(by_page=True)): - process_batch(page) - """ if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.records.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) - self._validate_constraints() + + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute() to prevent accidental full-table scans." + ) + params = self.build() client = self._query_ops._client - pages = client.records.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + if use_by_page: + return self.execute_pages() + + all_records: List[Record] = [] + with client._scoped_odata() as od: + for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + all_records.extend(Record.from_api_response(params["table"], row) for row in page) + return QueryResult(all_records) + + # ---------------------------------------------------------- execute_pages + + def execute_pages(self) -> Iterator[QueryResult]: + """Lazily yield one :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + per HTTP page. + + Each iteration triggers a network request via ``@odata.nextLink``. + One-shot — do not iterate more than once. + + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``execute_pages()``; otherwise a :class:`ValueError` is + raised to prevent accidental full-table scans. + + :return: Iterator of per-page :class:`~PowerPlatform.Dataverse.models.record.QueryResult`. + :rtype: Iterator[QueryResult] + :raises ValueError: If no ``select``, ``where``, or ``top`` + constraint has been set. + :raises RuntimeError: If the query was not created via + ``client.query.builder()``. - if by_page: - return pages + Example:: - def _flat() -> Iterable[Record]: - for page in pages: - yield from page + from PowerPlatform.Dataverse.models.filters import col - return _flat() + for page in (client.query.builder("account") + .select("name") + .where(col("statecode") == 0) + .execute_pages()): + process(page.to_dataframe()) + """ + if self._query_ops is None: + raise RuntimeError( + "Cannot execute: query was not created via client.query.builder(). " + "Use build() and pass parameters to client.records.list() instead." + ) + + if not self._select and not self._filter_parts and self._top is None and self._page_size is None: + raise ValueError( + "At least one of select(), where(), top(), or page_size() must be called before " + "execute_pages() to prevent accidental full-table scans." + ) + + params = self.build() + client = self._query_ops._client + + with client._scoped_odata() as od: + for page in od._get_multiple( + params["table"], + select=params.get("select"), + filter=params.get("filter"), + orderby=params.get("orderby"), + top=params.get("top"), + expand=params.get("expand"), + page_size=params.get("page_size"), + count=params.get("count", False), + include_annotations=params.get("include_annotations"), + ): + yield QueryResult([Record.from_api_response(params["table"], row) for row in page]) # ----------------------------------------------------------- to_dataframe def to_dataframe(self) -> pd.DataFrame: """Execute the query and return results as a pandas DataFrame. - All pages are consolidated into a single DataFrame, matching - the behavior of ``client.dataframe.get()``. + .. deprecated:: + Use ``QueryBuilder.execute().to_dataframe()`` instead. + ``QueryBuilder.to_dataframe()`` will be removed in a future release. + + All pages are consolidated into a single DataFrame. This method is only available when the QueryBuilder was created via ``client.query.builder(table)``. - At least one of ``select()``, ``filter_*()``, ``where()``, or - ``top()`` must be called before ``to_dataframe()``; otherwise a - :class:`ValueError` is raised to prevent accidental full-table - scans. + At least one of ``select()``, ``where()``, or ``top()`` must be + called before ``to_dataframe()``; otherwise a :class:`ValueError` + is raised to prevent accidental full-table scans. :return: DataFrame containing all matching records. Returns an empty DataFrame when no records match. :rtype: ~pandas.DataFrame - :raises ValueError: If no ``select``, ``filter``, or ``top`` + :raises ValueError: If no ``select``, ``where``, or ``top`` constraint has been set. :raises RuntimeError: If the query was not created via ``client.query.builder()``. Example:: + from PowerPlatform.Dataverse.models.filters import col + df = (client.query.builder("account") .select("name", "telephone1") - .filter_eq("statecode", 0) + .where(col("statecode") == 0) .top(100) + .execute() .to_dataframe()) """ + warnings.warn( + "'QueryBuilder.to_dataframe()' is deprecated; use " "'QueryBuilder.execute().to_dataframe()' instead.", + DeprecationWarning, + stacklevel=2, + ) if self._query_ops is None: raise RuntimeError( "Cannot execute: query was not created via client.query.builder(). " - "Use build() and pass parameters to client.dataframe.get() instead." + "Use build() and pass parameters to client.records.list() instead." ) - self._validate_constraints() - params = self.build() - return self._query_ops._client.dataframe.get( - params["table"], - select=params.get("select"), - filter=params.get("filter"), - orderby=params.get("orderby"), - top=params.get("top"), - expand=params.get("expand"), - page_size=params.get("page_size"), - count=params.get("count", False), - include_annotations=params.get("include_annotations"), - ) + + result = self.execute() + if not result: + return pd.DataFrame(columns=self._select) if self._select else pd.DataFrame() + return result.to_dataframe() diff --git a/src/PowerPlatform/Dataverse/models/record.py b/src/PowerPlatform/Dataverse/models/record.py index 43641032..e81f412c 100644 --- a/src/PowerPlatform/Dataverse/models/record.py +++ b/src/PowerPlatform/Dataverse/models/record.py @@ -6,9 +6,9 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, Dict, Iterator, KeysView, Optional, ValuesView, ItemsView +from typing import Any, Dict, Iterator, KeysView, List, Optional, ValuesView, ItemsView -__all__ = ["Record"] +__all__ = ["Record", "QueryResult"] _ODATA_PREFIX = "@odata." @@ -112,3 +112,55 @@ def from_api_response( def to_dict(self) -> Dict[str, Any]: """Return a plain dict copy of the record data (excludes metadata).""" return dict(self.data) + + +class QueryResult: + """Iterable wrapper around a list of :class:`Record` objects. + + Returned by :meth:`~PowerPlatform.Dataverse.models.query_builder.QueryBuilder.execute` + (flat mode) and :meth:`~PowerPlatform.Dataverse.operations.records.RecordOperations.list`. + + Backward-compatible: ``for r in result`` continues to work without change. + + :param records: Collected records from all pages. + :type records: list[:class:`Record`] + """ + + def __init__(self, records: List[Record]) -> None: + self.records: List[Record] = records + + def __iter__(self) -> Iterator[Record]: + return iter(self.records) + + def __len__(self) -> int: + return len(self.records) + + def __bool__(self) -> bool: + return bool(self.records) + + def __repr__(self) -> str: + return f"QueryResult({len(self.records)} records)" + + def __getitem__(self, index): + result = self.records[index] + return QueryResult(result) if isinstance(index, slice) else result + + def first(self) -> Optional[Record]: + """Return the first record, or ``None`` if the result is empty.""" + return self.records[0] if self.records else None + + def to_dataframe(self) -> Any: + """Return all records as a pandas DataFrame. + + :raises ImportError: If pandas is not installed. + :rtype: ~pandas.DataFrame + """ + try: + import pandas as pd + except ImportError as exc: + raise ImportError("pandas is required for to_dataframe(). " "Install it with: pip install pandas") from exc + + if not self.records: + return pd.DataFrame() + rows = [r.data if hasattr(r, "data") else dict(r) for r in self.records] + return pd.DataFrame.from_records(rows) diff --git a/src/PowerPlatform/Dataverse/operations/batch.py b/src/PowerPlatform/Dataverse/operations/batch.py index b2751d93..062da4ab 100644 --- a/src/PowerPlatform/Dataverse/operations/batch.py +++ b/src/PowerPlatform/Dataverse/operations/batch.py @@ -5,7 +5,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Union +import warnings +from typing import TYPE_CHECKING, Any, Dict, List, Optional, Protocol, Union import pandas as pd @@ -18,6 +19,7 @@ _RecordUpdate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpsert, _TableCreate, _TableDelete, @@ -43,6 +45,7 @@ if TYPE_CHECKING: from ..client import DataverseClient + from ..models.filters import FilterExpression __all__ = [ "BatchRecordOperations", @@ -56,6 +59,29 @@ ] +# --------------------------------------------------------------------------- +# Shared interface for batch operation namespaces +# --------------------------------------------------------------------------- + + +class _BatchContext(Protocol): + """Structural interface required by batch operation namespaces. + + The operation namespaces (BatchRecordOperations, BatchTableOperations, etc.) + are pure (no I/O) and shared by both the sync and async batch implementations. + This Protocol allows them to type-annotate their ``batch`` parameter correctly + without importing either concrete class (``BatchRequest`` or + ``AsyncBatchRequest``), which would otherwise require ``# type: ignore``. + + Both :class:`~PowerPlatform.Dataverse.operations.batch.BatchRequest` and + :class:`~PowerPlatform.Dataverse.aio.operations.async_batch.AsyncBatchRequest` + satisfy this protocol structurally — no explicit inheritance needed. + """ + + _items: List[Any] + records: Any # used by BatchDataFrameOperations to delegate create/update/delete + + # --------------------------------------------------------------------------- # Changeset namespaces # --------------------------------------------------------------------------- @@ -166,15 +192,18 @@ class BatchRecordOperations: """ Record operations on a :class:`BatchRequest`. - Mirrors ``client.records`` exactly: same method names, same signatures. + Mirrors ``client.records``: same method names, same signatures. All methods return ``None``; results are available via :class:`~PowerPlatform.Dataverse.models.batch.BatchResult` after :meth:`BatchRequest.execute`. + GA methods: :meth:`retrieve` (single record) and :meth:`list` (multi-record, + single page). :meth:`get` is deprecated — use :meth:`retrieve` instead. + Do not instantiate directly; use ``batch.records``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create( @@ -253,16 +282,9 @@ def get( """ Add a single-record get operation to the batch. - Only the single-record overload (``record_id`` provided) is supported. - The paginated/multi-record overload of ``client.records.get()`` - (``filter``, ``orderby``, etc., without ``record_id``) is **not** - supported in batch — pagination requires following - ``@odata.nextLink`` across multiple round-trips, which is - incompatible with a single batch request. - - The response body will be available in - :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` - after :meth:`BatchRequest.execute`. + .. deprecated:: + Use :meth:`retrieve` instead. ``batch.records.get()`` is deprecated + and will be removed in a future release. :param table: Table schema name. :type table: :class:`str` @@ -271,6 +293,11 @@ def get( :param select: Optional list of column names to include. :type select: list[str] or None """ + warnings.warn( + "'batch.records.get()' is deprecated; use 'batch.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) self._batch._items.append(_RecordGet(table=table, record_id=record_id, select=select)) def upsert( @@ -325,6 +352,136 @@ def upsert( raise TypeError("Each item must be an UpsertItem or a dict with 'alternate_key' and 'record' keys") self._batch._items.append(_RecordUpsert(table=table, items=normalized)) + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> None: + """ + Add a single-record retrieve operation to the batch. + + GA replacement for the deprecated :meth:`get`. Enqueues a GET request + for one record by its GUID. The response body will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand. + Navigation property names are case-sensitive and must match the + entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + + Example:: + + batch = client.batch.new() + batch.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + result = batch.execute() + record = result.responses[0].data + contact = (record.get("primarycontactid") or {}) + print(contact.get("fullname")) + """ + self._batch._items.append( + _RecordGet( + table=table, + record_id=record_id, + select=select, + expand=expand, + include_annotations=include_annotations, + ) + ) + + def list( + self, + table: str, + *, + filter: "Optional[Union[str, FilterExpression]]" = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> None: + """ + Add a multi-record list operation to the batch (single page, no pagination). + + Enqueues a GET request for multiple records. Because batch requests are + a single HTTP round-trip, pagination (``@odata.nextLink``) is **not** + supported — use ``top`` to bound the result size, or rely on the + server's default page limit. + + The response body (``{"value": [...]}`` JSON) will be available in + :attr:`~PowerPlatform.Dataverse.models.batch.BatchItemResponse.data` + after :meth:`BatchRequest.execute`. + + :param table: Table schema name (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData ``$filter`` expression or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc"]``). + :type orderby: list[str] or None + :param top: Maximum number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to the request. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + + Example:: + + batch = client.batch.new() + batch.records.list( + "account", + filter="statecode eq 0", + select=["name", "statuscode"], + orderby=["name asc"], + top=50, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + result = batch.execute() + records = result.responses[0].data.get("value", []) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + self._batch._items.append( + _RecordList( + table=table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + ) + class BatchTableOperations: """ @@ -348,7 +505,7 @@ class BatchTableOperations: Do not instantiate directly; use ``batch.tables``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create( @@ -586,7 +743,7 @@ class BatchQueryOperations: Do not instantiate directly; use ``batch.query``. """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def sql(self, sql: str) -> None: @@ -640,7 +797,7 @@ class BatchDataFrameOperations: result = batch.execute() """ - def __init__(self, batch: "BatchRequest") -> None: + def __init__(self, batch: "_BatchContext") -> None: self._batch = batch def create(self, table: str, records: pd.DataFrame) -> None: diff --git a/src/PowerPlatform/Dataverse/operations/dataframe.py b/src/PowerPlatform/Dataverse/operations/dataframe.py index e6ec2033..28647c58 100644 --- a/src/PowerPlatform/Dataverse/operations/dataframe.py +++ b/src/PowerPlatform/Dataverse/operations/dataframe.py @@ -5,6 +5,7 @@ from __future__ import annotations +import warnings from typing import TYPE_CHECKING, Any, Dict, List, Optional import pandas as pd @@ -164,6 +165,12 @@ def get( df = client.dataframe.get("account", select=["name"], top=100) """ + warnings.warn( + "'dataframe.get()' is deprecated; use " + "client.query.builder(table).where(...).execute().to_dataframe() instead.", + DeprecationWarning, + stacklevel=2, + ) if record_id is not None: if not isinstance(record_id, str) or not record_id.strip(): raise ValueError("record_id must be a non-empty string") @@ -173,25 +180,30 @@ def get( "Cannot specify query parameters (filter, orderby, top, " "expand, page_size) when fetching a single record by ID" ) - result = self._client.records.get( - table, - record_id, - select=select, - ) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self._client.records.get( + table, + record_id, + select=select, + ) return pd.DataFrame([result.data]) rows: List[dict] = [] - for batch in self._client.records.get( - table, - select=select, - filter=filter, - orderby=orderby, - top=top, - expand=expand, - page_size=page_size, - count=count, - include_annotations=include_annotations, - ): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = self._client.records.get( + table, + select=select, + filter=filter, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ) + for batch in pages: rows.extend(row.data for row in batch) if not rows: diff --git a/src/PowerPlatform/Dataverse/operations/query.py b/src/PowerPlatform/Dataverse/operations/query.py index 83e82677..c927bdf0 100644 --- a/src/PowerPlatform/Dataverse/operations/query.py +++ b/src/PowerPlatform/Dataverse/operations/query.py @@ -5,9 +5,13 @@ from __future__ import annotations +import warnings +import xml.etree.ElementTree as _ET from typing import Any, Dict, List, Optional, TYPE_CHECKING +from urllib.parse import quote as _url_quote -from ..core.errors import MetadataError +from ..core.errors import MetadataError, ValidationError +from ..models.fetchxml_query import FetchXmlQuery, _MAX_URL_LENGTH from ..models.record import Record from ..models.query_builder import QueryBuilder @@ -146,6 +150,80 @@ def sql(self, sql: str) -> List[Record]: rows = od._query_sql(sql) return [Record.from_api_response("", row) for row in rows] + # --------------------------------------------------------------- fetchxml + + def fetchxml(self, xml: str) -> FetchXmlQuery: + """Return an inert :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` object. + + No HTTP request is made until + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute` + or + :meth:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery.execute_pages` + is called on the returned object. + + Use for SQL-JOIN scenarios, aggregate queries, or other operations that + the OData builder endpoint cannot express. + + :param xml: Well-formed FetchXML query string. The root ```` + element determines the entity set endpoint. + :type xml: :class:`str` + :return: Inert query object with ``.execute()`` and ``.execute_pages()`` methods. + :rtype: :class:`~PowerPlatform.Dataverse.models.fetchxml_query.FetchXmlQuery` + :raises ValueError: If the FetchXML is missing a root ```` element + or the entity ``name`` attribute. + + Example:: + + query = client.query.fetchxml(\"\"\" + + + + + + + + + \"\"\") + + # Eager — collect all pages: + result = query.execute() + df = result.to_dataframe() + + # Lazy — process one page at a time: + for page in query.execute_pages(): + process(page.to_dataframe()) + """ + if not isinstance(xml, str): + raise ValidationError("xml must be a string") + xml = xml.strip() + if not xml: + raise ValidationError("xml must not be empty") + # Fast-fail before any HTTP is attempted; execute_pages() re-checks the full URL + # (base + encoded XML) on each page. + if len(_url_quote(xml, safe="")) > _MAX_URL_LENGTH: + raise ValidationError( + f"FetchXML exceeds the Dataverse URL length limit ({_MAX_URL_LENGTH:,} characters) when encoded. " + "Use a $batch POST request to send FetchXML in the request body where the limit is 64 KB." + ) + # Parse only to verify well-formedness and extract the entity name needed for the + # request URL. Structural and semantic validation is intentionally left to the server + # to avoid duplicating rules that may diverge from Dataverse's own enforcement. + # ElementTree does not resolve external entities or expand recursive internal entity + # references, so pathological inputs of that kind raise ParseError rather than + # consuming resources. + try: + root_el = _ET.fromstring(xml) + except _ET.ParseError as exc: + raise ValidationError(f"xml is not well-formed: {exc}") from exc + entity_el = root_el.find("entity") + if entity_el is None: + raise ValueError("FetchXML must contain an child element") + entity_name = entity_el.get("name", "") + if not entity_name: + raise ValueError("FetchXML element must have a 'name' attribute") + return FetchXmlQuery(xml, entity_name, self._client) + # --------------------------------------------------------------- sql_columns def sql_columns( @@ -230,178 +308,6 @@ def sql_columns( result.sort(key=lambda x: (not x["is_pk"], not x["is_name"], x["name"])) return result - # --------------------------------------------------------------- sql_select - - def sql_select( - self, - table: str, - *, - include_system: bool = False, - ) -> str: - """Return a comma-separated column list for use in SQL SELECT. - - Excludes virtual columns and optionally system columns. The result - can be embedded directly in a SQL query string. - - :param table: Schema name of the table (e.g. ``"account"``). - :type table: :class:`str` - :param include_system: Include system columns (default ``False``). - :type include_system: :class:`bool` - - :return: Comma-separated column names. - :rtype: :class:`str` - - Example:: - - cols = client.query.sql_select("account") - sql = f"SELECT TOP 10 {cols} FROM account" - df = client.dataframe.sql(sql) - """ - columns = self.sql_columns(table, include_system=include_system) - return ", ".join(c["name"] for c in columns) - - # --------------------------------------------------------------- sql_joins - - def sql_joins( - self, - table: str, - ) -> List[Dict[str, Any]]: - """Discover all possible SQL JOINs from a table. - - Returns one entry per outgoing lookup relationship, with the - exact column names needed for SQL ``JOIN ... ON`` clauses. - - For **polymorphic** lookups (e.g. ``customerid`` targeting both - ``account`` and ``contact``), multiple entries are returned with - the same ``column`` but different ``target`` values. - - :param table: Schema name of the table (e.g. ``"contact"``). - :type table: :class:`str` - - :return: List of JOIN metadata dicts, each containing: - - - ``column`` -- the lookup attribute on this table (use in ON clause) - - ``target`` -- the referenced entity name - - ``target_pk`` -- the referenced entity's primary key column - - ``relationship`` -- the schema name of the relationship - - ``join_clause`` -- a ready-to-use ``JOIN ... ON ...`` fragment - - :rtype: list[dict[str, typing.Any]] - - .. note:: - - The ``join_clause`` value references the source table by its - **full name** (e.g. ``ON contact.col = ...``), so the FROM - clause must also use the unaliased table name. For queries - that need aliases, use :meth:`sql_join` instead. - - Example:: - - joins = client.query.sql_joins("contact") - for j in joins: - print(f"{j['column']:30s} -> {j['target']}.{j['target_pk']}") - print(f" {j['join_clause']}") - - # Use in a query (no alias on the FROM table) - j = next(j for j in joins if j['target'] == 'account') - sql = f"SELECT TOP 10 contact.fullname, a.name FROM contact {j['join_clause']}" - """ - table_lower = table.lower() - rels = self._client.tables.list_table_relationships(table) - - used_aliases: set = set() - result: List[Dict[str, Any]] = [] - for r in rels: - ref_entity = (r.get("ReferencingEntity") or "").lower() - if ref_entity != table_lower: - continue - col = r.get("ReferencingAttribute", "") - target = r.get("ReferencedEntity", "") - target_pk = r.get("ReferencedAttribute", "") - schema = r.get("SchemaName", "") - if not all([col, target, target_pk]): - continue - - # Generate a unique alias — add a numeric suffix on collision so - # two lookups to tables starting with the same letter (e.g. - # "account" and "annotation") or two lookups to the same table - # (e.g. "ownerid" and "createdby" both to "systemuser") produce - # distinct aliases and valid SQL. - base = target[0] if target else "j" - alias = base - counter = 2 - while alias in used_aliases: - alias = f"{base}{counter}" - counter += 1 - used_aliases.add(alias) - join_clause = f"JOIN {target} {alias} ON {table_lower}.{col} = {alias}.{target_pk}" - - result.append( - { - "column": col, - "target": target, - "target_pk": target_pk, - "relationship": schema, - "join_clause": join_clause, - } - ) - - result.sort(key=lambda x: (x["target"], x["column"])) - return result - - # --------------------------------------------------------------- sql_join - - def sql_join( - self, - from_table: str, - to_table: str, - *, - from_alias: Optional[str] = None, - to_alias: Optional[str] = None, - ) -> str: - """Generate a SQL JOIN clause between two tables. - - Discovers the relationship automatically via metadata. If multiple - relationships exist (e.g. polymorphic lookups), picks the first - match. Use :meth:`sql_joins` to see all options. - - :param from_table: Schema name of the FROM table (e.g. ``"contact"``). - :type from_table: :class:`str` - :param to_table: Schema name of the target table (e.g. ``"account"``). - :type to_table: :class:`str` - :param from_alias: Optional alias for the FROM table in the JOIN - clause. If ``None``, uses the full table name. - :type from_alias: :class:`str` or None - :param to_alias: Optional alias for the target table. If ``None``, - uses the first letter of the target table name. - :type to_alias: :class:`str` or None - - :return: A ready-to-use ``JOIN ... ON ...`` clause. - :rtype: :class:`str` - - :raises ValueError: If no relationship is found between the tables. - - Example:: - - j = client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - # Returns: "JOIN account a ON c.parentcustomerid = a.accountid" - sql = f"SELECT TOP 10 c.fullname, a.name FROM contact c {j}" - df = client.dataframe.sql(sql) - """ - to_lower = to_table.lower() - joins = self.sql_joins(from_table) - match = [j for j in joins if j["target"].lower() == to_lower] - if not match: - raise ValueError( - f"No relationship found from '{from_table}' to '{to_table}'. " - f"Use client.query.sql_joins('{from_table}') to see available targets." - ) - - j = match[0] - src = from_alias or from_table.lower() - tgt = to_alias or to_lower[0] - return f"JOIN {to_lower} {tgt} " f"ON {src}.{j['column']} = {tgt}.{j['target_pk']}" - # =========================================================== # OData helpers -- eliminate friction for records.get() users # =========================================================== @@ -433,6 +339,12 @@ def odata_select( for r in page: print(r) """ + warnings.warn( + "'odata_select' is deprecated; use the typed builder (1.x) " + "or client.query.sql_columns() to discover columns.", + DeprecationWarning, + stacklevel=2, + ) columns = self.sql_columns(table, include_system=include_system) return [c["name"] for c in columns] @@ -545,6 +457,12 @@ def odata_expand( acct = r.get(nav) or {} print(f"{r['fullname']} -> {acct.get('name', 'N/A')}") """ + warnings.warn( + "'odata_expand' is deprecated; use the typed builder (1.x) " + "with .expand() or client.query.odata_expands() to discover navigation properties.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower] @@ -594,6 +512,11 @@ def odata_bind( **bind, }) """ + warnings.warn( + "'odata_bind' is deprecated; use the typed builder (1.x) " "or pass the @odata.bind dict manually.", + DeprecationWarning, + stacklevel=2, + ) to_lower = to_table.lower() expands = self.odata_expands(from_table) match = [e for e in expands if e["target_table"].lower() == to_lower and e["target_entity_set"]] diff --git a/src/PowerPlatform/Dataverse/operations/records.py b/src/PowerPlatform/Dataverse/operations/records.py index ef867f52..c9c66119 100644 --- a/src/PowerPlatform/Dataverse/operations/records.py +++ b/src/PowerPlatform/Dataverse/operations/records.py @@ -5,12 +5,15 @@ from __future__ import annotations -from typing import Any, Dict, Iterable, List, Optional, Union, overload, TYPE_CHECKING +import warnings +from typing import Any, Dict, Iterable, Iterator, List, Optional, Union, overload, TYPE_CHECKING -from ..models.record import Record +from ..core.errors import HttpError +from ..models.record import QueryResult, Record from ..models.upsert import UpsertItem if TYPE_CHECKING: + from ..models.filters import FilterExpression from ..client import DataverseClient @@ -151,6 +154,7 @@ def update( [id1, id2], [{"name": "Name A"}, {"name": "Name B"}], ) + """ with self._client._scoped_odata() as od: if isinstance(ids, str): @@ -395,10 +399,12 @@ def get( Only used for multi-record queries. :type include_annotations: :class:`str` or None - :return: A single record dict when ``record_id`` is provided, or a - generator yielding pages (lists of record dicts) when fetching - multiple records. - :rtype: dict or collections.abc.Iterable[list[dict]] + :return: A single :class:`~PowerPlatform.Dataverse.models.record.Record` + when ``record_id`` is provided, or a generator yielding pages + (lists of :class:`~PowerPlatform.Dataverse.models.record.Record`) + when fetching multiple records. + :rtype: ~PowerPlatform.Dataverse.models.record.Record or + collections.abc.Iterable[list[~PowerPlatform.Dataverse.models.record.Record]] :raises TypeError: If ``record_id`` is provided but not a string. :raises ValueError: If query parameters are provided alongside @@ -424,6 +430,12 @@ def get( print(record["name"]) """ if record_id is not None: + warnings.warn( + "'records.get()' with a record_id is deprecated; " + "use 'client.records.retrieve(table, record_id)' instead.", + DeprecationWarning, + stacklevel=2, + ) if not isinstance(record_id, str): raise TypeError("record_id must be str") if ( @@ -444,6 +456,12 @@ def get( raw = od._get(table, record_id, select=select) return Record.from_api_response(table, raw, record_id=record_id) + warnings.warn( + "'records.get()' is deprecated; " "use 'client.records.list(table, filter=...)' instead.", + DeprecationWarning, + stacklevel=2, + ) + def _paged() -> Iterable[List[Record]]: with self._client._scoped_odata() as od: for page in od._get_multiple( @@ -461,6 +479,202 @@ def _paged() -> Iterable[List[Record]]: return _paged() + # --------------------------------------------------------------- retrieve + + def retrieve( + self, + table: str, + record_id: str, + *, + select: Optional[List[str]] = None, + expand: Optional[List[str]] = None, + include_annotations: Optional[str] = None, + ) -> Optional[Record]: + """Fetch a single record by its GUID, returning ``None`` if not found. + + GA replacement for ``records.get(table, record_id)``. Returns ``None`` + instead of raising when the record does not exist (HTTP 404). + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param record_id: GUID of the record to retrieve. + :type record_id: :class:`str` + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param expand: Optional list of navigation properties to expand (e.g. + ``["primarycontactid"]``). Navigation property names are + case-sensitive and must match the entity's ``$metadata``. + :type expand: list[str] or None + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header (e.g. ``"*"`` or + ``"OData.Community.Display.V1.FormattedValue"``), or ``None``. + :type include_annotations: :class:`str` or None + :return: Typed record, or ``None`` if not found. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.Record` or None + + Example:: + + record = client.records.retrieve( + "account", account_id, + select=["name", "statuscode"], + expand=["primarycontactid"], + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + if record is not None: + contact = record.get("primarycontactid") or {} + print(contact.get("fullname")) + """ + with self._client._scoped_odata() as od: + try: + raw = od._get(table, record_id, select=select, expand=expand, include_annotations=include_annotations) + except HttpError as exc: + if exc.status_code == 404: + return None + raise + return Record.from_api_response(table, raw, record_id=record_id) + + # ------------------------------------------------------------------ list + + def list( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> QueryResult: + """Fetch multiple records and return them as a :class:`QueryResult`. + + GA replacement for ``records.get(table, filter=...)``. All pages are + collected eagerly and returned as a single :class:`QueryResult`. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: All matching records collected into a :class:`QueryResult`. + :rtype: :class:`~PowerPlatform.Dataverse.models.record.QueryResult` + + Example:: + + from PowerPlatform.Dataverse import col + + result = client.records.list( + "account", + filter=col("statecode") == 0, + select=["name", "statuscode"], + orderby=["name asc"], + top=100, + include_annotations="OData.Community.Display.V1.FormattedValue", + ) + for record in result: + print(record["name"], record.get("statuscode@OData.Community.Display.V1.FormattedValue")) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + all_records: List[Record] = [] + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + all_records.extend(Record.from_api_response(table, row) for row in page) + return QueryResult(all_records) + + # --------------------------------------------------------------- list_pages + + def list_pages( + self, + table: str, + *, + filter: Optional[Union[str, "FilterExpression"]] = None, + select: Optional[List[str]] = None, + orderby: Optional[List[str]] = None, + top: Optional[int] = None, + expand: Optional[List[str]] = None, + page_size: Optional[int] = None, + count: bool = False, + include_annotations: Optional[str] = None, + ) -> Iterator[QueryResult]: + """Lazily yield one :class:`QueryResult` per HTTP page. + + Streaming counterpart to :meth:`list`. Each iteration triggers one + network request via ``@odata.nextLink``. One-shot — do not iterate + more than once. + + :param table: Schema name of the table (e.g. ``"account"``). + :type table: :class:`str` + :param filter: Optional OData filter string or :class:`FilterExpression`. + :type filter: str or FilterExpression or None + :param select: Optional list of column logical names to include. + :type select: list[str] or None + :param orderby: Optional list of sort expressions (e.g. ``["name asc", "createdon desc"]``). + :type orderby: list[str] or None + :param top: Maximum total number of records to return. + :type top: int or None + :param expand: Optional list of navigation properties to expand. + :type expand: list[str] or None + :param page_size: Per-page size hint via ``Prefer: odata.maxpagesize``. + :type page_size: int or None + :param count: If ``True``, adds ``$count=true`` to include a total record count. + :type count: bool + :param include_annotations: OData annotation pattern for the + ``Prefer: odata.include-annotations`` header, or ``None``. + :type include_annotations: :class:`str` or None + :return: Iterator of per-page :class:`QueryResult` objects. + :rtype: Iterator[:class:`~PowerPlatform.Dataverse.models.record.QueryResult`] + + Example:: + + for page in client.records.list_pages( + "account", + filter="statecode eq 0", + orderby=["name asc"], + page_size=200, + ): + process(page.to_dataframe()) + """ + filter_str: Optional[str] = str(filter) if filter is not None else None + with self._client._scoped_odata() as od: + for page in od._get_multiple( + table, + select=select, + filter=filter_str, + orderby=orderby, + top=top, + expand=expand, + page_size=page_size, + count=count, + include_annotations=include_annotations, + ): + yield QueryResult([Record.from_api_response(table, row) for row in page]) + # ------------------------------------------------------------------ upsert def upsert(self, table: str, items: List[Union[UpsertItem, Dict[str, Any]]]) -> None: diff --git a/tests/unit/aio/__init__.py b/tests/unit/aio/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/unit/aio/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/unit/aio/conftest.py b/tests/unit/aio/conftest.py new file mode 100644 index 00000000..ab29d7c9 --- /dev/null +++ b/tests/unit/aio/conftest.py @@ -0,0 +1,35 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Shared fixtures for async unit tests.""" + +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock + +import pytest +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient + + +@pytest.fixture +def mock_od() -> AsyncMock: + """AsyncMock representing the low-level _AsyncODataClient.""" + od = AsyncMock() + # _call_scope() is a sync context manager; MagicMock supports __enter__/__exit__ + od._call_scope.return_value = MagicMock() + return od + + +@pytest.fixture +def async_client(mock_od: AsyncMock) -> AsyncDataverseClient: + """AsyncDataverseClient with _scoped_odata patched to yield mock_od.""" + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped_odata(): + yield mock_od + + client._scoped_odata = _fake_scoped_odata + return client diff --git a/tests/unit/aio/core/__init__.py b/tests/unit/aio/core/__init__.py new file mode 100644 index 00000000..9a045456 --- /dev/null +++ b/tests/unit/aio/core/__init__.py @@ -0,0 +1,2 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. diff --git a/tests/unit/aio/core/test_async_auth.py b/tests/unit/aio/core/test_async_auth.py new file mode 100644 index 00000000..6b425826 --- /dev/null +++ b/tests/unit/aio/core/test_async_auth.py @@ -0,0 +1,49 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.core._async_auth import _AsyncAuthManager +from PowerPlatform.Dataverse.core._auth import _TokenPair + + +class TestAsyncAuthManager: + """Tests for _AsyncAuthManager credential validation and token acquisition.""" + + def test_non_async_token_credential_raises(self): + """_AsyncAuthManager raises TypeError when credential does not implement AsyncTokenCredential.""" + with pytest.raises(TypeError) as exc_info: + _AsyncAuthManager("not-a-credential") + assert "AsyncTokenCredential" in str(exc_info.value) + + def test_valid_credential_accepted(self): + """_AsyncAuthManager accepts a valid AsyncTokenCredential.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + manager = _AsyncAuthManager(mock_cred) + assert manager.credential is mock_cred + + async def test_acquire_token_returns_token_pair(self): + """_acquire_token calls get_token and returns a _TokenPair with scope and token.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + mock_cred.get_token = AsyncMock(return_value=MagicMock(token="my-access-token")) + + manager = _AsyncAuthManager(mock_cred) + result = await manager._acquire_token("https://org.crm.dynamics.com/.default") + + mock_cred.get_token.assert_called_once_with("https://org.crm.dynamics.com/.default") + assert isinstance(result, _TokenPair) + assert result.resource == "https://org.crm.dynamics.com/.default" + assert result.access_token == "my-access-token" + + async def test_acquire_token_different_scope(self): + """_acquire_token passes the scope string through to get_token.""" + mock_cred = MagicMock(spec=AsyncTokenCredential) + mock_cred.get_token = AsyncMock(return_value=MagicMock(token="tok")) + + manager = _AsyncAuthManager(mock_cred) + await manager._acquire_token("https://example.crm10.dynamics.com/.default") + + mock_cred.get_token.assert_called_once_with("https://example.crm10.dynamics.com/.default") diff --git a/tests/unit/aio/core/test_async_http.py b/tests/unit/aio/core/test_async_http.py new file mode 100644 index 00000000..43708aba --- /dev/null +++ b/tests/unit/aio/core/test_async_http.py @@ -0,0 +1,286 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock, call, patch + +import aiohttp + +from PowerPlatform.Dataverse.aio.core._async_http import _AsyncHttpClient, _AsyncResponse + + +def _make_resp(status: int = 200) -> MagicMock: + """Return a mock aiohttp.ClientResponse.""" + resp = MagicMock() + resp.status = status + resp.headers = {} + resp.read = AsyncMock(return_value=b"") + return resp + + +def _make_cm(resp=None, exc=None) -> MagicMock: + """Return an async context manager mock. + + If exc is given, __aenter__ raises it. Otherwise it returns resp. + """ + cm = MagicMock() + if exc is not None: + cm.__aenter__ = AsyncMock(side_effect=exc) + else: + cm.__aenter__ = AsyncMock(return_value=resp) + cm.__aexit__ = AsyncMock(return_value=False) + return cm + + +def _make_session(status: int = 200) -> MagicMock: + """Return a mock aiohttp.ClientSession whose request() is an async context manager.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.request = MagicMock(return_value=_make_cm(_make_resp(status))) + return session + + +class TestAsyncHttpClientTimeout: + """Tests for automatic timeout selection in _AsyncHttpClient._request.""" + + async def test_get_uses_10s_default_timeout(self): + """GET requests use 10 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("get", "https://example.com/data") + _, kwargs = session.request.call_args + assert isinstance(kwargs["timeout"], aiohttp.ClientTimeout) + assert kwargs["timeout"].total == 10 + + async def test_post_uses_120s_default_timeout(self): + """POST requests use 120 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("post", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 120 + + async def test_delete_uses_120s_default_timeout(self): + """DELETE requests use 120 s default when no timeout is specified.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("delete", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 120 + + async def test_put_uses_10s_default_timeout(self): + """PUT requests use 10 s default (only POST/DELETE get 120 s).""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("put", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 10 + + async def test_patch_uses_10s_default_timeout(self): + """PATCH requests use 10 s default (only POST/DELETE get 120 s).""" + session = _make_session() + client = _AsyncHttpClient(retries=1, session=session) + await client._request("patch", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 10 + + async def test_custom_client_timeout_overrides_method_default(self): + """Explicit default_timeout on the client overrides per-method defaults.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, timeout=30.0, session=session) + await client._request("get", "https://example.com/data") + _, kwargs = session.request.call_args + assert kwargs["timeout"].total == 30.0 + + async def test_explicit_timeout_kwarg_takes_precedence(self): + """If timeout is already in kwargs it is passed through unchanged.""" + session = _make_session() + client = _AsyncHttpClient(retries=1, timeout=30.0, session=session) + custom_timeout = aiohttp.ClientTimeout(total=5) + await client._request("get", "https://example.com/data", timeout=custom_timeout) + _, kwargs = session.request.call_args + assert kwargs["timeout"] is custom_timeout + + +class TestAsyncHttpClientNoSession: + """Tests for RuntimeError when no session is provided.""" + + async def test_raises_runtime_error_without_session(self): + """_request raises RuntimeError if no session has been set.""" + client = _AsyncHttpClient(retries=1) + with pytest.raises(RuntimeError, match="No aiohttp.ClientSession"): + await client._request("get", "https://example.com") + + +class TestAsyncHttpClientRetry: + """Tests for retry behavior on aiohttp.ClientError.""" + + async def test_retries_on_client_error_and_succeeds(self): + """Retries after a ClientError and returns response on second attempt.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError("timeout")), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=2, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + result = await client._request("get", "https://example.com/data") + + assert session.request.call_count == 2 + assert isinstance(result, _AsyncResponse) + assert result.status == 200 + + async def test_raises_after_all_retries_exhausted(self): + """Raises ClientError after all retry attempts fail.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.request = MagicMock(return_value=_make_cm(exc=aiohttp.ClientConnectionError("timeout"))) + client = _AsyncHttpClient(retries=3, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(aiohttp.ClientError): + await client._request("get", "https://example.com/data") + + async def test_backoff_delay_between_retries(self): + """Sleeps with exponential backoff between retry attempts.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=3, backoff=1.0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await client._request("get", "https://example.com/data") + # First retry: 1.0 * 2^0 = 1.0; second retry: 1.0 * 2^1 = 2.0 + mock_sleep.assert_has_calls([call(1.0), call(2.0)]) + + async def test_no_retry_on_success(self): + """Single successful response does not trigger retries.""" + session = _make_session(200) + client = _AsyncHttpClient(retries=5, backoff=0, session=session) + await client._request("get", "https://example.com/data") + assert session.request.call_count == 1 + + async def test_retries_on_timeout_error(self): + """Retries on asyncio.TimeoutError (not a subclass of aiohttp.ClientError).""" + import asyncio + + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=asyncio.TimeoutError()), + _make_cm(good_resp), + ] + ) + client = _AsyncHttpClient(retries=2, backoff=0, session=session) + with patch("asyncio.sleep", new_callable=AsyncMock): + result = await client._request("get", "https://example.com/data") + + assert session.request.call_count == 2 + assert isinstance(result, _AsyncResponse) + assert result.status == 200 + + +class TestAsyncHttpClientClose: + """Tests for _AsyncHttpClient.close().""" + + async def test_close_closes_session(self): + """close() closes the session and sets _session to None.""" + session = MagicMock(spec=aiohttp.ClientSession) + session.close = AsyncMock() + client = _AsyncHttpClient(retries=1, session=session) + await client.close() + session.close.assert_called_once() + assert client._session is None + + async def test_close_without_session_is_safe(self): + """close() is safe to call when no session was set.""" + client = _AsyncHttpClient(retries=1) + await client.close() # should not raise + + +class TestAsyncHttpClientLogger: + """Tests for request logging via _HttpLogger integration.""" + + async def test_request_logged_when_logger_set(self): + """Outbound request is logged once when a logger is attached.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + mock_logger.log_request.assert_called_once() + + async def test_response_logged_when_logger_set(self): + """HTTP response is logged when a logger is attached.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + mock_logger.log_response.assert_called_once() + + async def test_error_logged_on_retry(self): + """Transport errors are logged before each retry.""" + session = MagicMock(spec=aiohttp.ClientSession) + good_resp = _make_resp(200) + session.request = MagicMock( + side_effect=[ + _make_cm(exc=aiohttp.ClientConnectionError()), + _make_cm(good_resp), + ] + ) + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=2, backoff=0, session=session, logger=mock_logger) + with patch("asyncio.sleep", new_callable=AsyncMock): + await client._request("get", "https://example.com/data") + mock_logger.log_error.assert_called_once() + + async def test_request_body_logged_from_json_kwarg(self): + """json= kwarg body is extracted and passed to log_request.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("post", "https://example.com/data", json={"key": "value"}) + _, log_kwargs = mock_logger.log_request.call_args + assert log_kwargs["body"] == {"key": "value"} + + async def test_request_body_logged_from_data_kwarg(self): + """data= kwarg body is extracted when json= is absent.""" + session = _make_session() + mock_logger = MagicMock() + mock_logger.body_logging_enabled = False + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("post", "https://example.com/data", data=b"raw bytes") + _, log_kwargs = mock_logger.log_request.call_args + assert log_kwargs["body"] == b"raw bytes" + + async def test_response_body_decoded_when_body_logging_enabled(self): + """When body_logging_enabled=True, response bytes are decoded and passed to log_response.""" + session = _make_session() + session.request.return_value.__aenter__.return_value.read = AsyncMock(return_value=b'{"ok": true}') + mock_logger = MagicMock() + mock_logger.body_logging_enabled = True + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + _, log_kwargs = mock_logger.log_response.call_args + assert log_kwargs["body"] == '{"ok": true}' + + async def test_response_body_invalid_bytes_replaced_in_logging(self): + """Invalid UTF-8 bytes in response body are replaced (not raised) when body logging is enabled.""" + session = _make_session() + session.request.return_value.__aenter__.return_value.read = AsyncMock(return_value=b"\xff\xfe invalid") + mock_logger = MagicMock() + mock_logger.body_logging_enabled = True + client = _AsyncHttpClient(retries=1, session=session, logger=mock_logger) + await client._request("get", "https://example.com/data") + _, log_kwargs = mock_logger.log_response.call_args + # errors="replace" means invalid bytes become replacement chars — body is a str, never raises + assert isinstance(log_kwargs["body"], str) diff --git a/tests/unit/aio/data/__init__.py b/tests/unit/aio/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/aio/data/test_async_batch_internal.py b/tests/unit/aio/data/test_async_batch_internal.py new file mode 100644 index 00000000..1a24deed --- /dev/null +++ b/tests/unit/aio/data/test_async_batch_internal.py @@ -0,0 +1,839 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncBatchClient internals.""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_batch import _AsyncBatchClient +from PowerPlatform.Dataverse.aio.core._async_http import _AsyncResponse +from PowerPlatform.Dataverse.core.errors import MetadataError, ValidationError +from PowerPlatform.Dataverse.data._batch_base import ( + _RecordCreate, + _RecordDelete, + _RecordGet, + _RecordList, + _RecordUpdate, + _RecordUpsert, + _TableAddColumns, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _TableRemoveColumns, + _QuerySql, + _ChangeSet, + _MAX_BATCH_SIZE, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_batch_client(): + """Return _AsyncBatchClient with a fully-mocked _AsyncODataClient. + + All _build_* methods are pre-mocked so resolver tests can run without + any real OData or HTTP logic. Sync _build_* methods use MagicMock; + async ones use AsyncMock. + """ + od = AsyncMock() + od.api = "https://example.crm.dynamics.com/api/data/v9.2" + od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + od._primary_id_attr = AsyncMock(return_value="accountid") + od._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account", "SchemaName": "Account"} + ) + od._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1", "LogicalName": "new_notes"}) + od._build_create = AsyncMock( + return_value=MagicMock(method="POST", url="https://x/accounts", body="{}", headers=None, content_id=None) + ) + od._build_create_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/CreateMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_update = AsyncMock( + return_value=MagicMock( + method="PATCH", url="https://x/accounts(g)", body="{}", headers={"If-Match": "*"}, content_id=None + ) + ) + od._build_update_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/UpdateMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_delete = AsyncMock( + return_value=MagicMock( + method="DELETE", url="https://x/accounts(g)", body=None, headers={"If-Match": "*"}, content_id=None + ) + ) + od._build_delete_multiple = AsyncMock( + return_value=MagicMock(method="POST", url="https://x/BulkDelete", body="{}", headers=None, content_id=None) + ) + od._build_get = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts(g)", body=None, headers=None, content_id=None) + ) + od._build_upsert = AsyncMock( + return_value=MagicMock(method="PATCH", url="https://x/accounts(k)", body="{}", headers=None, content_id=None) + ) + od._build_upsert_multiple = AsyncMock( + return_value=MagicMock( + method="POST", url="https://x/accounts/UpsertMultiple", body="{}", headers=None, content_id=None + ) + ) + od._build_list = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts", body=None, headers=None, content_id=None) + ) + od._build_sql = AsyncMock( + return_value=MagicMock(method="GET", url="https://x/accounts?sql=...", body=None, headers=None, content_id=None) + ) + od._build_delete_entity = MagicMock( + return_value=MagicMock( + method="DELETE", url="https://x/EntityDefinitions(m)", body=None, headers=None, content_id=None + ) + ) + od._build_create_column = MagicMock( + return_value=MagicMock( + method="POST", url="https://x/EntityDefinitions(m)/Attributes", body="{}", headers=None, content_id=None + ) + ) + od._build_delete_column = MagicMock( + return_value=MagicMock( + method="DELETE", + url="https://x/EntityDefinitions(m)/Attributes(a)", + body=None, + headers=None, + content_id=None, + ) + ) + # Sync _build_* for pure-logic table intents inherited from _BatchBase + _raw = lambda method, url: MagicMock(method=method, url=url, body=None, headers=None, content_id=None) + od._build_create_entity = MagicMock(return_value=_raw("POST", "https://x/EntityDefinitions")) + od._build_get_entity = MagicMock(return_value=_raw("GET", "https://x/EntityDefinitions(m)")) + od._build_list_entities = MagicMock(return_value=_raw("GET", "https://x/EntityDefinitions")) + od._build_create_relationship = MagicMock(return_value=_raw("POST", "https://x/RelationshipDefinitions")) + od._build_delete_relationship = MagicMock(return_value=_raw("DELETE", "https://x/RelationshipDefinitions(r)")) + od._build_get_relationship = MagicMock(return_value=_raw("GET", "https://x/RelationshipDefinitions(r)")) + _mock_lookup = MagicMock() + _mock_lookup.to_dict.return_value = {} + _mock_rel = MagicMock() + _mock_rel.to_dict.return_value = {} + od._build_lookup_field_models = MagicMock(return_value=(_mock_lookup, _mock_rel)) + return _AsyncBatchClient(od), od + + +def _batch_resp(status=200, text="", json_payload=None): + """Create a mock _AsyncResponse-compatible response for the batch execute() path.""" + r = MagicMock() + r.status = status + r.status_code = status + r.headers = {"Content-Type": "application/json"} + r.text = text + r.json = MagicMock(return_value=json_payload or {}) + return r + + +# --------------------------------------------------------------------------- +# execute() +# --------------------------------------------------------------------------- + + +class TestExecute: + """Tests for execute(), the public entry point that dispatches the full batch request.""" + + async def test_empty_items_returns_empty_result(self): + """An empty items list short-circuits and returns an empty BatchResult without HTTP.""" + client, _ = _make_batch_client() + result = await client.execute([]) + assert result is not None + + async def test_executes_single_record_create(self): + """A single RecordCreate item causes exactly one _request call.""" + client, od = _make_batch_client() + from PowerPlatform.Dataverse.models.batch import BatchResult + + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()): + await client.execute([item]) + od._request.assert_called_once() + + async def test_executes_with_continue_on_error(self): + """The odata.continue-on-error preference is injected when continue_on_error=True.""" + client, od = _make_batch_client() + from PowerPlatform.Dataverse.models.batch import BatchResult + + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()): + await client.execute([item], continue_on_error=True) + call_kwargs = od._request.call_args.kwargs + headers = call_kwargs.get("headers", {}) + assert "odata.continue-on-error" in headers.get("Prefer", "") + + +# --------------------------------------------------------------------------- +# _resolve_record_create() +# --------------------------------------------------------------------------- + + +class TestResolveRecordCreate: + """Tests for _resolve_record_create() intent-to-request translation.""" + + async def test_single_dict_returns_one_request(self): + """A dict payload produces a single _build_create request.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data={"name": "X"}) + result = await client._resolve_record_create(op) + assert len(result) == 1 + od._build_create.assert_called_once() + + async def test_list_returns_one_create_multiple_request(self): + """A list payload produces a single _build_create_multiple request.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data=[{"name": "X"}, {"name": "Y"}]) + result = await client._resolve_record_create(op) + assert len(result) == 1 + od._build_create_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_update() +# --------------------------------------------------------------------------- + + +class TestResolveRecordUpdate: + """Tests for _resolve_record_update() intent-to-request translation.""" + + async def test_single_id_string_returns_one_patch(self): + """A single string ID with a dict of changes produces one _build_update request.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids="guid-1", changes={"name": "X"}) + result = await client._resolve_record_update(op) + assert len(result) == 1 + od._build_update.assert_called_once() + + async def test_single_id_non_dict_changes_raises(self): + """TypeError is raised when changes is not a dict for a single-ID update.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids="guid-1", changes=["invalid"]) + with pytest.raises(TypeError): + await client._resolve_record_update(op) + + async def test_list_ids_returns_update_multiple(self): + """A list of IDs produces a single _build_update_multiple request.""" + client, od = _make_batch_client() + op = _RecordUpdate(table="account", ids=["id-1", "id-2"], changes={"name": "X"}) + result = await client._resolve_record_update(op) + assert len(result) == 1 + od._build_update_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_delete() +# --------------------------------------------------------------------------- + + +class TestResolveRecordDelete: + """Tests for _resolve_record_delete() intent-to-request translation.""" + + async def test_single_id_string(self): + """A single string ID produces one _build_delete request.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids="guid-1") + result = await client._resolve_record_delete(op) + assert len(result) == 1 + od._build_delete.assert_called_once() + + async def test_list_ids_bulk_delete(self): + """A list of IDs with use_bulk_delete=True produces one _build_delete_multiple request.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=True) + result = await client._resolve_record_delete(op) + assert len(result) == 1 + od._build_delete_multiple.assert_called_once() + + async def test_list_ids_sequential_delete(self): + """A list of IDs with use_bulk_delete=False produces one _build_delete per ID.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=False) + result = await client._resolve_record_delete(op) + assert len(result) == 2 + assert od._build_delete.call_count == 2 + + async def test_empty_ids_list_returns_empty(self): + """An empty IDs list produces no requests.""" + client, od = _make_batch_client() + op = _RecordDelete(table="account", ids=[]) + result = await client._resolve_record_delete(op) + assert result == [] + + +# --------------------------------------------------------------------------- +# _resolve_record_get() +# --------------------------------------------------------------------------- + + +class TestResolveRecordGet: + """Tests for _resolve_record_get() intent-to-request translation.""" + + async def test_single_get_request(self): + """A RecordGet op produces one _build_get request with the correct arguments.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1", select=["name"]) + result = await client._resolve_record_get(op) + assert len(result) == 1 + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=None + ) + + async def test_passes_expand_to_build_get(self): + """expand= is forwarded from _RecordGet to _build_get.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1", expand=["primarycontactid"]) + await client._resolve_record_get(op) + od._build_get.assert_called_once_with( + "account", "guid-1", select=None, expand=["primarycontactid"], include_annotations=None + ) + + async def test_passes_include_annotations_to_build_get(self): + """include_annotations= is forwarded from _RecordGet to _build_get.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet(table="account", record_id="guid-1", include_annotations=annotation) + await client._resolve_record_get(op) + od._build_get.assert_called_once_with( + "account", "guid-1", select=None, expand=None, include_annotations=annotation + ) + + async def test_passes_all_params_to_build_get(self): + """All _RecordGet fields are forwarded together to _build_get.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet( + table="account", + record_id="guid-1", + select=["name"], + expand=["primarycontactid"], + include_annotations=annotation, + ) + result = await client._resolve_record_get(op) + assert len(result) == 1 + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=["primarycontactid"], include_annotations=annotation + ) + + +# --------------------------------------------------------------------------- +# _resolve_record_list() +# --------------------------------------------------------------------------- + + +class TestResolveRecordList: + """Tests for _resolve_record_list() intent-to-request translation.""" + + async def test_produces_one_request(self): + """A _RecordList op produces exactly one _build_list request.""" + client, od = _make_batch_client() + op = _RecordList(table="account") + result = await client._resolve_record_list(op) + assert len(result) == 1 + od._build_list.assert_called_once() + + async def test_passes_table_to_build_list(self): + """The table name is forwarded to _build_list as first positional arg.""" + client, od = _make_batch_client() + op = _RecordList(table="contact") + await client._resolve_record_list(op) + call_args = od._build_list.call_args + assert call_args[0][0] == "contact" + + async def test_passes_filter(self): + """filter= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", filter="statecode eq 0") + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["filter"] == "statecode eq 0" + + async def test_passes_select(self): + """select= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", select=["name", "revenue"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["select"] == ["name", "revenue"] + + async def test_passes_top(self): + """top= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", top=50) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["top"] == 50 + + async def test_passes_orderby(self): + """orderby= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", orderby=["name asc"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["orderby"] == ["name asc"] + + async def test_passes_expand(self): + """expand= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", expand=["primarycontactid"]) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["expand"] == ["primarycontactid"] + + async def test_passes_page_size(self): + """page_size= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", page_size=200) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["page_size"] == 200 + + async def test_passes_count(self): + """count=True is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + op = _RecordList(table="account", count=True) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["count"] is True + + async def test_passes_include_annotations(self): + """include_annotations= is forwarded from _RecordList to _build_list.""" + client, od = _make_batch_client() + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordList(table="account", include_annotations=annotation) + await client._resolve_record_list(op) + assert od._build_list.call_args[1]["include_annotations"] == annotation + + async def test_resolve_item_dispatch(self): + """_resolve_item dispatches _RecordList correctly.""" + client, od = _make_batch_client() + op = _RecordList(table="account", filter="statecode eq 0") + result = await client._resolve_item(op) + assert len(result) == 1 + od._build_list.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_record_upsert() +# --------------------------------------------------------------------------- + + +class TestResolveRecordUpsert: + """Tests for _resolve_record_upsert() intent-to-request translation.""" + + async def test_single_item_calls_build_upsert(self): + """A single UpsertItem produces one _build_upsert request.""" + client, od = _make_batch_client() + item = UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "X"}) + op = _RecordUpsert(table="account", items=[item]) + result = await client._resolve_record_upsert(op) + assert len(result) == 1 + od._build_upsert.assert_called_once() + + async def test_multiple_items_calls_build_upsert_multiple(self): + """Multiple UpsertItems produce a single _build_upsert_multiple request.""" + client, od = _make_batch_client() + items = [ + UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "X"}), + UpsertItem(alternate_key={"accountnumber": "B"}, record={"name": "Y"}), + ] + op = _RecordUpsert(table="account", items=items) + result = await client._resolve_record_upsert(op) + assert len(result) == 1 + od._build_upsert_multiple.assert_called_once() + + +# --------------------------------------------------------------------------- +# _resolve_table_delete() +# --------------------------------------------------------------------------- + + +class TestResolveTableDelete: + """Tests for _resolve_table_delete() intent-to-request translation.""" + + async def test_resolves_to_delete_request(self): + """A TableDelete op resolves to a _build_delete_entity call using the table's MetadataId.""" + client, od = _make_batch_client() + op = _TableDelete(table="account") + result = await client._resolve_table_delete(op) + assert len(result) == 1 + od._build_delete_entity.assert_called_once_with("meta-1") + + async def test_table_not_found_raises(self): + """MetadataError is raised when the target table does not exist in metadata.""" + client, od = _make_batch_client() + od._get_entity_by_table_schema_name = AsyncMock(return_value=None) + op = _TableDelete(table="nonexistent") + with pytest.raises(MetadataError): + await client._resolve_table_delete(op) + + +# --------------------------------------------------------------------------- +# _resolve_table_add_columns() +# --------------------------------------------------------------------------- + + +class TestResolveTableAddColumns: + """Tests for _resolve_table_add_columns() intent-to-request translation.""" + + async def test_resolves_to_create_column_requests(self): + """Each column in the op produces one _build_create_column request.""" + client, od = _make_batch_client() + op = _TableAddColumns(table="account", columns={"col1": "string", "col2": "decimal"}) + result = await client._resolve_table_add_columns(op) + assert len(result) == 2 + assert od._build_create_column.call_count == 2 + + +# --------------------------------------------------------------------------- +# _resolve_table_remove_columns() +# --------------------------------------------------------------------------- + + +class TestResolveTableRemoveColumns: + """Tests for _resolve_table_remove_columns() intent-to-request translation.""" + + async def test_resolves_to_delete_column_requests(self): + """Each column in the list produces one _build_delete_column request.""" + client, od = _make_batch_client() + op = _TableRemoveColumns(table="account", columns=["col1", "col2"]) + result = await client._resolve_table_remove_columns(op) + assert len(result) == 2 + + async def test_string_column_name(self): + """A single column name supplied as a string produces one delete request.""" + client, od = _make_batch_client() + op = _TableRemoveColumns(table="account", columns="col1") + result = await client._resolve_table_remove_columns(op) + assert len(result) == 1 + + async def test_column_not_found_raises(self): + """MetadataError is raised when attribute metadata returns None for the column.""" + client, od = _make_batch_client() + od._get_attribute_metadata = AsyncMock(return_value=None) + op = _TableRemoveColumns(table="account", columns=["nonexistent"]) + with pytest.raises(MetadataError): + await client._resolve_table_remove_columns(op) + + async def test_attr_missing_metadata_id_raises(self): + """MetadataError is raised when attribute metadata lacks a MetadataId field.""" + client, od = _make_batch_client() + od._get_attribute_metadata = AsyncMock(return_value={"LogicalName": "col1"}) + op = _TableRemoveColumns(table="account", columns=["col1"]) + with pytest.raises(MetadataError): + await client._resolve_table_remove_columns(op) + + +# --------------------------------------------------------------------------- +# _resolve_query_sql() +# --------------------------------------------------------------------------- + + +class TestResolveQuerySql: + """Tests for _resolve_query_sql() intent-to-request translation.""" + + async def test_resolves_to_get_request(self): + """A QuerySql op produces one _build_sql request with the SQL statement.""" + client, od = _make_batch_client() + op = _QuerySql(sql="SELECT name FROM account") + result = await client._resolve_query_sql(op) + assert len(result) == 1 + od._build_sql.assert_called_once_with("SELECT name FROM account") + + +# --------------------------------------------------------------------------- +# _resolve_one() — changeset item must produce exactly 1 request +# --------------------------------------------------------------------------- + + +class TestResolveOne: + """Tests for _resolve_one(), which enforces the single-request contract for changeset items.""" + + async def test_single_request_returned(self): + """An op that resolves to exactly one request is returned without error.""" + client, od = _make_batch_client() + op = _RecordGet(table="account", record_id="guid-1") + req = await client._resolve_one(op) + assert req is not None + + async def test_multi_request_item_raises(self): + """ValidationError is raised when an op resolves to more than one request.""" + client, od = _make_batch_client() + # _RecordDelete with a list produces multiple requests (one per ID) + op = _RecordDelete(table="account", ids=["id-1", "id-2"], use_bulk_delete=False) + with pytest.raises(ValidationError, match="exactly one"): + await client._resolve_one(op) + + +# --------------------------------------------------------------------------- +# _resolve_all() — changeset handling +# --------------------------------------------------------------------------- + + +class TestResolveAll: + """Tests for _resolve_all(), which dispatches items and wraps changeset ops.""" + + async def test_empty_changeset_skipped(self): + """A ChangeSet with no operations is silently skipped without error.""" + client, od = _make_batch_client() + cs = _ChangeSet(operations=[]) + result = await client._resolve_all([cs]) + assert result == [] + + async def test_changeset_with_operations(self): + """A ChangeSet with one operation produces one _ChangeSetBatchItem in the result.""" + client, od = _make_batch_client() + op = _RecordCreate(table="account", data={"name": "X"}) + cs = _ChangeSet(operations=[op]) + result = await client._resolve_all([cs]) + assert len(result) == 1 + + async def test_unknown_item_type_raises(self): + """ValidationError is raised when an unrecognised item type is passed to _resolve_item.""" + client, od = _make_batch_client() + with pytest.raises(ValidationError, match="Unknown batch item type"): + await client._resolve_item("not-a-valid-type") + + +# --------------------------------------------------------------------------- +# _require_entity_metadata() +# --------------------------------------------------------------------------- + + +class TestRequireEntityMetadata: + """Tests for _require_entity_metadata(), which resolves a table's MetadataId or raises.""" + + async def test_returns_metadata_id(self): + """The MetadataId string is returned when the table exists in entity metadata.""" + client, od = _make_batch_client() + meta_id = await client._require_entity_metadata("account") + assert meta_id == "meta-1" + + async def test_not_found_raises(self): + """MetadataError is raised when the API returns no entity definition for the table.""" + client, od = _make_batch_client() + od._get_entity_by_table_schema_name = AsyncMock(return_value=None) + with pytest.raises(MetadataError): + await client._require_entity_metadata("nonexistent") + + +# --------------------------------------------------------------------------- +# _AsyncResponse +# --------------------------------------------------------------------------- + + +class TestHttpResponse: + """Tests for _AsyncResponse — the materialized response returned by _AsyncHttpClient.""" + + def test_json_parses_body(self): + """json() parses the body bytes as JSON.""" + r = _AsyncResponse(200, {"Content-Type": "application/json"}, b'{"value": []}') + assert r.json() == {"value": []} + + def test_json_empty_body_returns_empty_dict(self): + """json() returns {} for an empty body.""" + r = _AsyncResponse(200, {}, b"") + assert r.json() == {} + + def test_status_and_status_code_equal(self): + """status and status_code are both set and equal.""" + r = _AsyncResponse(207, {}, b"") + assert r.status == 207 + assert r.status_code == 207 + + def test_text_decodes_body(self): + """text property decodes body bytes as UTF-8.""" + r = _AsyncResponse(200, {}, b"body text") + assert r.text == "body text" + + def test_text_empty_body(self): + """text property returns empty string for empty body.""" + r = _AsyncResponse(200, {}, b"") + assert r.text == "" + + +# --------------------------------------------------------------------------- +# execute() edge cases +# --------------------------------------------------------------------------- + + +class TestExecuteEdgeCases: + """Tests for execute() error paths not covered by TestExecute.""" + + async def test_batch_size_exceeded_raises(self): + """execute() raises ValidationError when more than _MAX_BATCH_SIZE items are resolved.""" + client, od = _make_batch_client() + # _RecordCreate × (_MAX_BATCH_SIZE + 1) — each resolves to one request + items = [_RecordCreate(table="account", data={"name": f"X{i}"}) for i in range(_MAX_BATCH_SIZE + 1)] + with pytest.raises(ValidationError, match="exceeds the limit"): + await client.execute(items) + + async def test_response_passed_directly_to_parse_batch_response(self): + """execute() passes the HTTP response object directly to _parse_batch_response.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + client, od = _make_batch_client() + resp_mock = _batch_resp(status=200) + od._request = AsyncMock(return_value=resp_mock) + item = _RecordCreate(table="account", data={"name": "X"}) + with patch.object(client, "_parse_batch_response", return_value=BatchResult()) as mock_parse: + await client.execute([item]) + mock_parse.assert_called_once_with(resp_mock) + + +# --------------------------------------------------------------------------- +# _resolve_all() edge cases +# --------------------------------------------------------------------------- + + +class TestResolveAllEdgeCases: + """Tests for _resolve_all() paths not covered elsewhere.""" + + async def test_empty_changeset_is_skipped(self): + """An empty _ChangeSet is silently dropped from the resolved list.""" + client, od = _make_batch_client() + cs = _ChangeSet(_counter=[1]) + # No operations added — operations list is empty + result = await client._resolve_all([cs]) + assert result == [] + + async def test_non_changeset_item_extended(self): + """Non-changeset items are resolved and extended into the flat result.""" + client, od = _make_batch_client() + item = _RecordCreate(table="account", data={"name": "X"}) + result = await client._resolve_all([item]) + assert len(result) == 1 + + +# --------------------------------------------------------------------------- +# _resolve_item() full dispatch coverage +# --------------------------------------------------------------------------- + + +class TestResolveItemDispatch: + """One test per intent type — drives every branch of _resolve_item(). + + Each test replaces the specific resolver method with a mock so only the + dispatch logic is exercised. Record/SQL resolvers are async (awaited); + pure table resolvers inherited from _BatchBase are sync (not awaited). + """ + + _sentinel = MagicMock(method="GET", url="https://x/test", body=None, headers=None, content_id=None) + + def _async_mock(self): + return AsyncMock(return_value=[self._sentinel]) + + def _sync_mock(self): + return MagicMock(return_value=[self._sentinel]) + + async def test_dispatch_record_update(self): + client, _ = _make_batch_client() + client._resolve_record_update = self._async_mock() + result = await client._resolve_item(_RecordUpdate(table="account", ids="g", changes={"name": "X"})) + client._resolve_record_update.assert_called_once() + + async def test_dispatch_record_upsert(self): + client, _ = _make_batch_client() + client._resolve_record_upsert = self._async_mock() + item = UpsertItem(alternate_key={"k": "v"}, record={"name": "X"}) + result = await client._resolve_item(_RecordUpsert(table="account", items=[item])) + client._resolve_record_upsert.assert_called_once() + + async def test_dispatch_table_create(self): + client, _ = _make_batch_client() + client._resolve_table_create = self._sync_mock() + result = await client._resolve_item(_TableCreate(table="new_Test", columns={"new_Name": "string"})) + client._resolve_table_create.assert_called_once() + + async def test_dispatch_table_delete(self): + client, _ = _make_batch_client() + client._resolve_table_delete = self._async_mock() + result = await client._resolve_item(_TableDelete(table="new_Test")) + client._resolve_table_delete.assert_called_once() + + async def test_dispatch_table_get(self): + client, _ = _make_batch_client() + client._resolve_table_get = self._sync_mock() + result = await client._resolve_item(_TableGet(table="account")) + client._resolve_table_get.assert_called_once() + + async def test_dispatch_table_list(self): + client, _ = _make_batch_client() + client._resolve_table_list = self._sync_mock() + result = await client._resolve_item(_TableList()) + client._resolve_table_list.assert_called_once() + + async def test_dispatch_table_add_columns(self): + client, _ = _make_batch_client() + client._resolve_table_add_columns = self._async_mock() + result = await client._resolve_item(_TableAddColumns(table="account", columns={"new_X": "string"})) + client._resolve_table_add_columns.assert_called_once() + + async def test_dispatch_table_remove_columns(self): + client, _ = _make_batch_client() + client._resolve_table_remove_columns = self._async_mock() + result = await client._resolve_item(_TableRemoveColumns(table="account", columns="new_X")) + client._resolve_table_remove_columns.assert_called_once() + + async def test_dispatch_table_create_one_to_many(self): + client, _ = _make_batch_client() + client._resolve_table_create_one_to_many = self._sync_mock() + op = _TableCreateOneToMany(relationship=MagicMock(), lookup=MagicMock()) + result = await client._resolve_item(op) + client._resolve_table_create_one_to_many.assert_called_once() + + async def test_dispatch_table_create_many_to_many(self): + client, _ = _make_batch_client() + client._resolve_table_create_many_to_many = self._sync_mock() + op = _TableCreateManyToMany(relationship=MagicMock()) + result = await client._resolve_item(op) + client._resolve_table_create_many_to_many.assert_called_once() + + async def test_dispatch_table_delete_relationship(self): + client, _ = _make_batch_client() + client._resolve_table_delete_relationship = self._sync_mock() + result = await client._resolve_item(_TableDeleteRelationship(relationship_id="rel-guid")) + client._resolve_table_delete_relationship.assert_called_once() + + async def test_dispatch_table_get_relationship(self): + client, _ = _make_batch_client() + client._resolve_table_get_relationship = self._sync_mock() + result = await client._resolve_item(_TableGetRelationship(schema_name="new_a_c")) + client._resolve_table_get_relationship.assert_called_once() + + async def test_dispatch_table_create_lookup_field(self): + client, _ = _make_batch_client() + client._resolve_table_create_lookup_field = self._sync_mock() + result = await client._resolve_item( + _TableCreateLookupField( + referencing_table="contact", + lookup_field_name="new_accountid", + referenced_table="account", + ) + ) + client._resolve_table_create_lookup_field.assert_called_once() + + async def test_dispatch_query_sql(self): + client, _ = _make_batch_client() + client._resolve_query_sql = self._async_mock() + result = await client._resolve_item(_QuerySql(sql="SELECT accountid FROM account")) + client._resolve_query_sql.assert_called_once() + + async def test_dispatch_unknown_type_raises(self): + """An unrecognised intent type raises ValidationError.""" + client, _ = _make_batch_client() + with pytest.raises(ValidationError, match="Unknown batch item type"): + await client._resolve_item("not-an-intent") diff --git a/tests/unit/aio/data/test_async_odata_internal.py b/tests/unit/aio/data/test_async_odata_internal.py new file mode 100644 index 00000000..a064c1ed --- /dev/null +++ b/tests/unit/aio/data/test_async_odata_internal.py @@ -0,0 +1,1846 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncODataClient internals (mocking _request at the HTTP boundary).""" + +import json +import time +import warnings +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient +from PowerPlatform.Dataverse.core.errors import HttpError, MetadataError, ValidationError + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked out at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(json_data=None, status=200, headers=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.text = json.dumps(json_data) if json_data is not None else "" + r.json = MagicMock(return_value=json_data if json_data is not None else {}) + return r + + +def _entity_def( + entity_set="accounts", + pk="accountid", + meta_id="meta-001", + schema="Account", + logical="account", +): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": entity_set, + "PrimaryIdAttribute": pk, + "MetadataId": meta_id, + "SchemaName": schema, + } + ] + } + + +def _seed_cache(client: _AsyncODataClient, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +# --------------------------------------------------------------------------- +# close() +# --------------------------------------------------------------------------- + + +class TestClose: + """Tests for the close() lifecycle method.""" + + async def test_close_delegates_to_http(self): + """close() forwards to the underlying HTTP client's close() exactly once.""" + client = _make_client() + client._http.close = AsyncMock() + await client.close() + client._http.close.assert_called_once() + + async def test_close_clears_entity_set_cache(self): + """close() empties the entity-set lookup cache so stale entries don't persist.""" + client = _make_client() + _seed_cache(client) + client._http.close = AsyncMock() + await client.close() + assert len(client._logical_to_entityset_cache) == 0 + + +# --------------------------------------------------------------------------- +# _request() — tests actual implementation via _raw_request mock +# --------------------------------------------------------------------------- + + +class TestRequest: + """Tests for _request() error extraction. + + These tests mock _raw_request (one level below _request) so the real + header-building, status-checking, and error-parsing code runs. + """ + + def _auth_client(self): + """Return a client with a real auth mock but _raw_request not yet patched.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + return _AsyncODataClient(auth, "https://example.crm.dynamics.com") + + async def test_ok_response_returned(self): + """2xx responses are returned to the caller without raising.""" + client = self._auth_client() + client._raw_request = AsyncMock(return_value=_resp(status=200, json_data={"value": []})) + r = await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert r.status == 200 + + async def test_error_with_nested_error_object(self): + """Nested error.code / error.message body structure is parsed into HttpError.""" + client = self._auth_client() + body = {"error": {"code": "0x80040265", "message": "Not found"}} + client._raw_request = AsyncMock(return_value=_resp(status=404, json_data=body)) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 404 + assert "Not found" in str(exc.value) + + async def test_error_with_message_at_root(self): + """A top-level message key in the body is used when error nesting is absent.""" + client = self._auth_client() + body = {"message": "Root-level message"} + client._raw_request = AsyncMock(return_value=_resp(status=400, json_data=body)) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert "Root-level message" in str(exc.value) + + async def test_error_non_json_body_handled(self): + """Non-JSON response body falls back to HTTP status code as the error message.""" + client = self._auth_client() + r = MagicMock() + r.status = 503 + r.headers = {} + r.text = "Service Unavailable" + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 503 + + async def test_retry_after_header_parsed(self): + """Retry-After header value is stored as an integer in the error's details dict.""" + client = self._auth_client() + body = {"error": {"code": "429", "message": "Too many requests"}} + r = _resp(status=429, json_data=body, headers={"Retry-After": "60"}) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("retry_after") == 60 + + async def test_service_request_id_extracted(self): + """x-ms-service-request-id header is stored in the error's details dict.""" + client = self._auth_client() + r = _resp( + status=500, + json_data={"error": {"code": "err", "message": "fail"}}, + headers={"x-ms-service-request-id": "srv-req-1"}, + ) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("service_request_id") == "srv-req-1" + + +# --------------------------------------------------------------------------- +# _create() +# --------------------------------------------------------------------------- + + +class TestCreate: + """Tests for _create() single-record creation.""" + + async def test_returns_guid_from_odata_entity_id(self): + """GUID is extracted from the OData-EntityId response header.""" + client = _make_client() + _seed_cache(client) + guid = "12345678-1234-1234-1234-123456789012" + client._request.return_value = _resp( + status=204, + headers={"OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/accounts({guid})"}, + ) + result = await client._create("accounts", "account", {"amount": 100}) + assert result == guid + + async def test_returns_guid_from_location_header(self): + """Location header is used as fallback when OData-EntityId is absent.""" + client = _make_client() + _seed_cache(client) + guid = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + client._request.return_value = _resp( + status=204, + headers={"Location": f"https://example.crm.dynamics.com/api/data/v9.2/accounts({guid})"}, + ) + result = await client._create("accounts", "account", {"amount": 100}) + assert result == guid + + async def test_raises_when_no_guid_in_headers(self): + """RuntimeError is raised when neither OData-EntityId nor Location contains a GUID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204, headers={}) + with pytest.raises(RuntimeError, match="GUID"): + await client._create("accounts", "account", {"amount": 100}) + + +# --------------------------------------------------------------------------- +# _create_multiple() +# --------------------------------------------------------------------------- + + +class TestCreateMultiple: + """Tests for _create_multiple() bulk record creation.""" + + async def test_returns_ids_from_ids_key(self): + """IDs are extracted from the top-level Ids key in the response body.""" + client = _make_client() + _seed_cache(client) + guids = ["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"] + client._request.return_value = _resp(json_data={"Ids": guids}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}, {"amount": 2}]) + assert result == guids + + async def test_returns_ids_from_value_list(self): + """IDs are extracted from value list entries when the Ids key is absent.""" + client = _make_client() + _seed_cache(client) + guid = "11111111-1111-1111-1111-111111111111" + client._request.return_value = _resp(json_data={"value": [{"accountid": guid}]}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [guid] + + async def test_empty_body_returns_empty_list(self): + """An empty response body returns an empty list without raising.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={}, status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + async def test_non_dict_records_raises(self): + """TypeError is raised when the records list contains non-dict items.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError): + await client._create_multiple("accounts", "account", ["not-a-dict"]) + + +# --------------------------------------------------------------------------- +# _update() / _update_by_ids() / _update_multiple() +# --------------------------------------------------------------------------- + + +class TestUpdate: + """Tests for _update(), _update_by_ids(), and _update_multiple().""" + + async def test_update_patches_record(self): + """_update() issues a PATCH request for the given record ID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update("account", "guid-1", {"telephone1": "555"}) + assert client._request.called + + async def test_update_by_ids_broadcast_dict(self): + """A dict for changes is broadcast to all IDs via UpdateMultiple.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update_by_ids("account", ["id-1", "id-2"], {"statecode": 0}) + assert client._request.called + + async def test_update_by_ids_paired_list(self): + """A list for changes is applied pairwise with the corresponding IDs.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._update_by_ids("account", ["id-1"], [{"name": "A"}]) + assert client._request.called + + async def test_update_by_ids_empty_list_is_noop(self): + """An empty ID list short-circuits without issuing any HTTP request.""" + client = _make_client() + result = await client._update_by_ids("account", [], {"statecode": 0}) + assert result is None + client._request.assert_not_called() + + async def test_update_by_ids_mismatched_length_raises(self): + """ValueError is raised when the changes list is shorter than the ID list.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=200) + with pytest.raises(ValueError): + await client._update_by_ids("account", ["id-1", "id-2"], [{"name": "A"}]) + + async def test_update_by_ids_invalid_changes_type_raises(self): + """TypeError is raised when changes is neither a dict nor a list.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError): + await client._update_by_ids("account", ["id-1"], "invalid") + + async def test_update_multiple_empty_records_raises(self): + """TypeError is raised when the records list is empty.""" + client = _make_client() + with pytest.raises(TypeError): + await client._update_multiple("accounts", "account", []) + + async def test_update_multiple_non_list_raises(self): + """TypeError is raised when the records argument is not a list.""" + client = _make_client() + with pytest.raises(TypeError): + await client._update_multiple("accounts", "account", "not-a-list") + + +# --------------------------------------------------------------------------- +# _delete() / _delete_multiple() +# --------------------------------------------------------------------------- + + +class TestDelete: + """Tests for _delete() single-record deletion and _delete_multiple() bulk deletion.""" + + async def test_delete_calls_request(self): + """_delete() issues a DELETE request for the given record ID.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._delete("account", "guid-1") + assert client._request.called + + async def test_delete_multiple_returns_job_id(self): + """JobId from the async BulkDelete response is returned to the caller.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"JobId": "job-guid-1"}, status=202) + result = await client._delete_multiple("account", ["id-1", "id-2"]) + assert result == "job-guid-1" + + async def test_delete_multiple_empty_ids_returns_none(self): + """An empty ID list short-circuits without issuing any HTTP request.""" + client = _make_client() + result = await client._delete_multiple("account", []) + assert result is None + client._request.assert_not_called() + + async def test_delete_multiple_no_job_id_in_body(self): + """None is returned when the response body does not contain a JobId key.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={}, status=204) + result = await client._delete_multiple("account", ["id-1"]) + assert result is None + + +# --------------------------------------------------------------------------- +# _get() / _get_multiple() +# --------------------------------------------------------------------------- + + +class TestGet: + """Tests for _get() single-record fetch.""" + + async def test_get_returns_record(self): + """The full record dict from the response body is returned unchanged.""" + client = _make_client() + _seed_cache(client) + record = {"accountid": "guid-1", "name": "Contoso"} + client._request.return_value = _resp(json_data=record, status=200) + result = await client._get("account", "guid-1") + assert result == record + + async def test_get_with_select_param(self): + """The select list is forwarded as a $select query parameter.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"name": "Contoso"}, status=200) + result = await client._get("account", "guid-1", select=["name"]) + assert result == {"name": "Contoso"} + + +class TestGetMultiple: + """Tests for _get_multiple() async generator for paged results.""" + + async def test_single_page_yielded(self): + """A single-page response produces exactly one batch from the generator.""" + client = _make_client() + _seed_cache(client) + page = {"value": [{"accountid": "1"}, {"accountid": "2"}]} + client._request.return_value = _resp(json_data=page, status=200) + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 1 + assert len(pages[0]) == 2 + + async def test_follows_next_link(self): + """@odata.nextLink is followed to fetch subsequent pages automatically.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=xyz" + page1 = {"value": [{"accountid": "1"}], "@odata.nextLink": next_url} + page2 = {"value": [{"accountid": "2"}]} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 2 + + async def test_empty_value_not_yielded(self): + """A page with an empty value list produces no output from the generator.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"value": []}, status=200) + pages = [] + async for p in client._get_multiple("account"): + pages.append(p) + assert len(pages) == 0 + + async def test_with_all_params(self): + """All optional query parameters are forwarded in the outbound request.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data={"value": []}, status=200) + async for _ in client._get_multiple( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=10, + expand=["primarycontactid"], + page_size=5, + count=True, + include_annotations="*", + ): + pass + call = client._request.call_args + assert call is not None + kwargs = call.kwargs + assert "headers" in kwargs or kwargs.get("params") is not None + + +# --------------------------------------------------------------------------- +# _query_sql() +# --------------------------------------------------------------------------- + + +class TestQuerySql: + """Tests for _query_sql() which executes Dataverse SQL against the TDS endpoint.""" + + async def test_raises_if_not_string(self): + """ValidationError is raised when the SQL argument is not a string.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._query_sql(123) + + async def test_raises_if_empty(self): + """ValidationError is raised for a blank or whitespace-only SQL string.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._query_sql(" ") + + async def test_returns_rows_from_value(self): + """Rows are extracted from the value list in a standard OData response body.""" + client = _make_client() + _seed_cache(client) + rows = [{"name": "A"}, {"name": "B"}] + client._request.return_value = _resp(json_data={"value": rows}, status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == rows + + async def test_returns_list_body_directly(self): + """A list response body (rather than {value: [...]}) is accepted as rows directly.""" + client = _make_client() + _seed_cache(client) + rows = [{"name": "A"}] + client._request.return_value = _resp(json_data=rows, status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == rows + + async def test_follows_next_link(self): + """Pagination via @odata.nextLink concatenates all rows across pages.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name+FROM+account&page=2" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": next_url} + page2 = {"value": [{"name": "B"}]} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + result = await client._query_sql("SELECT name FROM account") + assert len(result) == 2 + + async def test_warns_and_stops_on_url_cycle(self): + """A repeated nextLink triggers a warning and stops pagination to prevent an infinite loop.""" + client = _make_client() + _seed_cache(client) + cycle_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name+FROM+account&page=1" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": cycle_url} + page2 = {"value": [{"name": "B"}], "@odata.nextLink": cycle_url} + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=page2)] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + # Cycle detected after page 2's nextLink repeats; pages 1 and 2 are collected. + assert any("same nextLink" in str(w.message) for w in caught) + assert len(result) == 2 + + async def test_stops_on_non_dict_page_body(self): + """A non-dict page body halts pagination and discards the malformed page.""" + client = _make_client() + _seed_cache(client) + next_url = "https://example.crm.dynamics.com/api/data/v9.2/accounts?sql=SELECT+name&page=2" + page1 = {"value": [{"name": "A"}], "@odata.nextLink": next_url} + # page2 is a list, not a dict — break condition + client._request.side_effect = [_resp(json_data=page1), _resp(json_data=["not-a-dict"])] + result = await client._query_sql("SELECT name FROM account") + assert len(result) == 1 + + +# --------------------------------------------------------------------------- +# _entity_set_from_schema_name() +# --------------------------------------------------------------------------- + + +class TestEntitySetResolution: + """Tests for _entity_set_from_schema_name() cache lookup and HTTP fetch.""" + + async def test_cache_hit_skips_http(self): + """A pre-populated cache entry is returned without any HTTP call.""" + client = _make_client() + _seed_cache(client) + result = await client._entity_set_from_schema_name("account") + client._request.assert_not_called() + assert result == "accounts" + + async def test_fetches_and_caches(self): + """On a cache miss, the entity set is fetched from the API and cached for reuse.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + result = await client._entity_set_from_schema_name("account") + assert result == "accounts" + # Subsequent call must hit the cache, not the API. + client._request.reset_mock() + result2 = await client._entity_set_from_schema_name("account") + client._request.assert_not_called() + assert result2 == "accounts" + + async def test_caches_primary_id_attr(self): + """The primary ID attribute is cached alongside the entity set name.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(pk="accountid"), status=200) + await client._entity_set_from_schema_name("account") + key = client._normalize_cache_key("account") + assert client._logical_primaryid_cache.get(key) == "accountid" + + async def test_not_found_raises_metadata_error(self): + """MetadataError is raised when the API returns an empty value list.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="Unable to resolve"): + await client._entity_set_from_schema_name("nonexistent") + + async def test_plural_name_includes_hint(self): + """The error message hints at a plural-name mistake when the input ends with 's'.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="plural"): + await client._entity_set_from_schema_name("accounts") + + async def test_missing_entity_set_name_raises(self): + """MetadataError is raised when the entity definition lacks an EntitySetName.""" + client = _make_client() + client._request.return_value = _resp( + json_data={"value": [{"LogicalName": "account", "MetadataId": "m1"}]}, + status=200, + ) + with pytest.raises(MetadataError, match="EntitySetName"): + await client._entity_set_from_schema_name("account") + + async def test_empty_name_raises_value_error(self): + """ValueError is raised immediately for an empty table schema name.""" + client = _make_client() + with pytest.raises(ValueError): + await client._entity_set_from_schema_name("") + + +# --------------------------------------------------------------------------- +# _get_table_info() / _list_tables() / _delete_table() +# --------------------------------------------------------------------------- + + +class TestTableInfo: + """Tests for _get_table_info() entity-definition summary lookup.""" + + async def test_get_table_info_found(self): + """A found table returns a dict containing entity_set_name and columns_created.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + result = await client._get_table_info("account") + assert result is not None + assert result["entity_set_name"] == "accounts" + assert result["columns_created"] == [] + + async def test_get_table_info_not_found(self): + """None is returned when the table does not exist in metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + result = await client._get_table_info("nonexistent") + assert result is None + + +class TestListTables: + """Tests for _list_tables() entity-definition list retrieval.""" + + async def test_list_tables_returns_value(self): + """The value list from the EntityDefinitions response is returned unchanged.""" + client = _make_client() + tables = [{"LogicalName": "account"}] + client._request.return_value = _resp(json_data={"value": tables}, status=200) + result = await client._list_tables() + assert result == tables + + async def test_list_tables_with_filter_and_select(self): + """Optional filter and select parameters are forwarded to the API request.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + result = await client._list_tables(filter="IsPrivate eq false", select=["LogicalName"]) + assert result == [] + + +class TestDeleteTable: + """Tests for _delete_table() metadata-level table removal.""" + + async def test_delete_calls_delete_request(self): + """Two requests are issued: one to resolve the MetadataId, one DELETE.""" + client = _make_client() + client._request.side_effect = [_resp(json_data=_entity_def()), _resp(status=204)] + await client._delete_table("account") + assert client._request.call_count == 2 + + async def test_delete_not_found_raises(self): + """MetadataError is raised when the target table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._delete_table("nonexistent") + + +# --------------------------------------------------------------------------- +# _create_table() +# --------------------------------------------------------------------------- + + +class TestCreateTable: + """Tests for _create_table() custom table provisioning.""" + + async def test_table_already_exists_raises(self): + """MetadataError is raised when a table with the same schema name already exists.""" + client = _make_client() + client._request.return_value = _resp(json_data=_entity_def(), status=200) + with pytest.raises(MetadataError, match="already exists"): + await client._create_table("account", {}) + + async def test_success_with_columns(self): + """Table and typed columns are created; the returned dict lists columns_created.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp( + json_data=_entity_def(entity_set="new_products", schema="new_Product", logical="new_product"), + status=200, + ) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {"new_Price": "decimal"}) + assert result["table_schema_name"] == "new_Product" + assert "new_Price" in result["columns_created"] + + async def test_success_with_primary_column(self): + """An explicit primary_column_schema_name is accepted without error.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp(json_data=_entity_def(entity_set="new_products"), status=200) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {}, primary_column_schema_name="new_ProductName") + assert result is not None + + async def test_success_with_display_name(self): + """A string display_name is accepted and forwarded to the API.""" + client = _make_client() + not_found = _resp(json_data={"value": []}, status=200) + create_resp = _resp(status=204) + entity_resp = _resp(json_data=_entity_def(entity_set="new_products"), status=200) + client._request.side_effect = [not_found, create_resp, entity_resp] + result = await client._create_table("new_Product", {}, display_name="Product") + assert result is not None + + async def test_unsupported_column_type_raises(self): + """ValueError is raised before the POST when a column type string is unrecognised.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(ValueError, match="Unsupported"): + await client._create_table("new_Product", {"col": "badtype"}) + + async def test_empty_solution_name_raises(self): + """ValueError is raised when solution_unique_name is an empty string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(ValueError, match="cannot be empty"): + await client._create_table("new_Product", {}, solution_unique_name="") + + async def test_non_string_solution_raises(self): + """TypeError is raised when solution_unique_name is not a string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(TypeError): + await client._create_table("new_Product", {}, solution_unique_name=42) + + async def test_invalid_display_name_raises(self): + """TypeError is raised when display_name is not a string.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(TypeError): + await client._create_table("new_Product", {}, display_name=123) + + +# --------------------------------------------------------------------------- +# _create_columns() / _delete_columns() +# --------------------------------------------------------------------------- + + +class TestCreateColumns: + """Tests for _create_columns() column provisioning on an existing table.""" + + async def test_creates_string_column(self): + """A string-typed column is created and its name returned in the result list.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp] + result = await client._create_columns("account", {"new_Notes": "string"}) + assert result == ["new_Notes"] + + async def test_empty_columns_dict_raises(self): + """TypeError is raised when the columns dict is empty.""" + client = _make_client() + with pytest.raises(TypeError, match="non-empty dict"): + await client._create_columns("account", {}) + + async def test_non_dict_columns_raises(self): + """TypeError is raised when the columns argument is not a dict.""" + client = _make_client() + with pytest.raises(TypeError): + await client._create_columns("account", ["col"]) + + async def test_table_not_found_raises(self): + """MetadataError is raised when the parent table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._create_columns("nonexistent", {"col": "string"}) + + async def test_unsupported_type_raises_validation_error(self): + """ValidationError is raised for an unrecognised column type string.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + client._request.return_value = entity_resp + with pytest.raises(ValidationError): + await client._create_columns("account", {"col": "badtype"}) + + async def test_optionset_column_flushes_cache(self): + """Creating a column whose payload includes OptionSet invalidates the picklist label cache. + + Boolean columns produce an OptionSet payload, which triggers the same cache-flush + path used by choice/picklist columns. + """ + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp] + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {"old": {}}} + result = await client._create_columns("account", {"new_Status": "bool"}) + assert result == ["new_Status"] + assert key not in client._picklist_label_cache + + +class TestDeleteColumns: + """Tests for _delete_columns() column removal from an existing table.""" + + async def test_string_column_name(self): + """A single column name supplied as a string is accepted and deleted.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_notes"}]}, status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + result = await client._delete_columns("account", "new_Notes") + assert result == ["new_Notes"] + + async def test_list_column_names(self): + """Column names supplied as a list are each deleted in turn.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_notes"}]}, status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + result = await client._delete_columns("account", ["new_Notes"]) + assert result == ["new_Notes"] + + async def test_invalid_type_raises(self): + """TypeError is raised when the columns argument is neither str nor list.""" + client = _make_client() + with pytest.raises(TypeError): + await client._delete_columns("account", 42) + + async def test_empty_column_name_raises(self): + """ValueError is raised when the column name string is empty.""" + client = _make_client() + with pytest.raises(ValueError, match="non-empty"): + await client._delete_columns("account", "") + + async def test_table_not_found_raises(self): + """MetadataError is raised when the parent table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._delete_columns("nonexistent", "col") + + async def test_column_not_found_raises(self): + """MetadataError is raised when the column is absent from attribute metadata.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": []}, status=200) + client._request.side_effect = [entity_resp, attr_resp] + with pytest.raises(MetadataError, match="not found"): + await client._delete_columns("account", "nonexistent_col") + + async def test_missing_attr_metadata_id_raises(self): + """RuntimeError is raised when the attribute response lacks a MetadataId.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp(json_data={"value": [{"LogicalName": "new_notes"}]}, status=200) + client._request.side_effect = [entity_resp, attr_resp] + with pytest.raises(RuntimeError, match="MetadataId"): + await client._delete_columns("account", "new_Notes") + + async def test_picklist_column_flushes_cache(self): + """Deleting a Picklist-type column invalidates the picklist label cache.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + attr_resp = _resp( + json_data={"value": [{"MetadataId": "attr-1", "LogicalName": "new_status", "AttributeType": "Picklist"}]}, + status=200, + ) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, attr_resp, delete_resp] + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + result = await client._delete_columns("account", "new_Status") + assert result == ["new_Status"] + assert key not in client._picklist_label_cache + + +# --------------------------------------------------------------------------- +# _list_columns() +# --------------------------------------------------------------------------- + + +class TestListColumns: + """Tests for _list_columns() attribute metadata listing.""" + + async def test_returns_attribute_list(self): + """The full attribute list from the API response is returned.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + cols_resp = _resp(json_data={"value": [{"LogicalName": "name"}, {"LogicalName": "accountid"}]}, status=200) + client._request.side_effect = [entity_resp, cols_resp] + result = await client._list_columns("account") + assert len(result) == 2 + + async def test_table_not_found_raises(self): + """MetadataError is raised when the table is absent from metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError, match="not found"): + await client._list_columns("nonexistent") + + async def test_with_select_and_filter(self): + """Optional select and filter parameters are forwarded to the Attributes API call.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + cols_resp = _resp(json_data={"value": []}, status=200) + client._request.side_effect = [entity_resp, cols_resp] + result = await client._list_columns("account", select=["LogicalName"], filter="AttributeType eq 'String'") + assert result == [] + + +# --------------------------------------------------------------------------- +# Alternate key operations +# --------------------------------------------------------------------------- + + +class TestAlternateKeys: + """Tests for _create_alternate_key(), _get_alternate_keys(), and _delete_alternate_key().""" + + async def test_create_alternate_key_success(self): + """The key UUID is extracted from the OData-EntityId header and returned in metadata_id. + + The URL format is EntityDefinitions(LogicalName='...')/Keys(uuid), so the regex + skips the LogicalName= form and matches only the key UUID in parentheses. + """ + client = _make_client() + key_uuid = "aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" + entity_resp = _resp(json_data=_entity_def(), status=200) + create_resp = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/EntityDefinitions(LogicalName='account')/Keys({key_uuid})" + }, + ) + client._request.side_effect = [entity_resp, create_resp] + result = await client._create_alternate_key("account", "new_prod_key", ["new_productcode"]) + assert result["schema_name"] == "new_prod_key" + assert result["key_attributes"] == ["new_productcode"] + assert result["metadata_id"] == key_uuid + + async def test_create_alternate_key_table_not_found_raises(self): + """MetadataError is raised when the target table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._create_alternate_key("nonexistent", "key", ["col"]) + + async def test_get_alternate_keys_returns_list(self): + """All alternate keys on the table are returned as a list of dicts.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + keys_resp = _resp(json_data={"value": [{"SchemaName": "key1"}, {"SchemaName": "key2"}]}, status=200) + client._request.side_effect = [entity_resp, keys_resp] + result = await client._get_alternate_keys("account") + assert len(result) == 2 + + async def test_get_alternate_keys_table_not_found_raises(self): + """MetadataError is raised when the table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._get_alternate_keys("nonexistent") + + async def test_delete_alternate_key_success(self): + """Two requests are issued: one entity lookup then one DELETE for the key.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def(), status=200) + delete_resp = _resp(status=204) + client._request.side_effect = [entity_resp, delete_resp] + await client._delete_alternate_key("account", "key-guid") + assert client._request.call_count == 2 + + async def test_delete_alternate_key_table_not_found_raises(self): + """MetadataError is raised when the table does not exist.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}, status=200) + with pytest.raises(MetadataError): + await client._delete_alternate_key("nonexistent", "key-guid") + + +# --------------------------------------------------------------------------- +# _upsert() / _upsert_multiple() +# --------------------------------------------------------------------------- + + +class TestUpsert: + """Tests for _upsert() and _upsert_multiple() alternate-key upsert operations.""" + + async def test_upsert_issues_patch(self): + """_upsert() issues a PATCH request (create-or-replace semantics).""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._upsert("accounts", "account", {"accountnumber": "A"}, {"name": "X"}) + call = client._request.call_args + assert call.args[0] == "patch" + + async def test_upsert_multiple_issues_post(self): + """_upsert_multiple() sends a POST to the UpsertMultiple action endpoint.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(status=204) + await client._upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}, {"accountnumber": "B"}], + [{"name": "X"}, {"name": "Y"}], + ) + call = client._request.call_args + assert call.args[0] == "post" + assert "UpsertMultiple" in call.args[1] + + async def test_upsert_multiple_mismatched_length_raises(self): + """ValueError is raised when the alternate-key list and record list differ in length.""" + client = _make_client() + with pytest.raises(ValueError, match="same length"): + await client._upsert_multiple("accounts", "account", [{"k": "1"}], [{"n": "A"}, {"n": "B"}]) + + async def test_upsert_multiple_key_conflict_raises(self): + """ValueError is raised when a record field conflicts with its alternate-key field.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValueError, match="conflicts"): + await client._upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"accountnumber": "B"}], + ) + + +# --------------------------------------------------------------------------- +# _bulk_fetch_picklists() / _convert_labels_to_ints() +# --------------------------------------------------------------------------- + + +class TestPicklists: + """Tests for _bulk_fetch_picklists() cache population and _convert_labels_to_ints() resolution.""" + + async def test_bulk_fetch_populates_cache(self): + """Picklist options are fetched from the API and stored with lowercased label keys.""" + client = _make_client() + body = { + "value": [ + { + "LogicalName": "statecode", + "OptionSet": { + "Options": [ + {"Value": 0, "Label": {"LocalizedLabels": [{"Label": "Active", "LanguageCode": 1033}]}}, + {"Value": 1, "Label": {"LocalizedLabels": [{"Label": "Inactive", "LanguageCode": 1033}]}}, + ] + }, + } + ] + } + client._request.return_value = _resp(json_data=body, status=200) + await client._bulk_fetch_picklists("account") + key = client._normalize_cache_key("account") + assert key in client._picklist_label_cache + picklists = client._picklist_label_cache[key]["picklists"] + assert "statecode" in picklists + assert picklists["statecode"]["active"] == 0 + + async def test_bulk_fetch_skips_on_cache_hit(self): + """A valid cache entry prevents an API call when the TTL has not expired.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + await client._bulk_fetch_picklists("account") + client._request.assert_not_called() + + async def test_bulk_fetch_empty_option_set(self): + """An attribute with an empty OptionSet is stored as an empty mapping.""" + client = _make_client() + body = {"value": [{"LogicalName": "field", "OptionSet": {}}]} + client._request.return_value = _resp(json_data=body, status=200) + await client._bulk_fetch_picklists("account") + key = client._normalize_cache_key("account") + assert client._picklist_label_cache[key]["picklists"]["field"] == {} + + async def test_convert_no_string_values_returns_unchanged(self): + """A record with no string values is returned as-is without any API lookup.""" + client = _make_client() + record = {"statecode": 0, "count": 5} + result = await client._convert_labels_to_ints("account", record) + assert result == record + client._request.assert_not_called() + + async def test_convert_string_resolved_to_int(self): + """A known label string is resolved to its integer option value from the cache.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"statecode": {"active": 0, "inactive": 1}}, + } + result = await client._convert_labels_to_ints("account", {"statecode": "Active"}) + assert result["statecode"] == 0 + + async def test_convert_odata_key_skipped(self): + """OData annotation fields with labels that don't match any option are left unchanged.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"@odata.type": {"val": 1}}, + } + record = {"@odata.type": "Microsoft.Dynamics.CRM.account"} + result = await client._convert_labels_to_ints("account", record) + assert result["@odata.type"] == "Microsoft.Dynamics.CRM.account" + + async def test_convert_unresolved_string_left_unchanged(self): + """A string value with no matching picklist entry is left as-is in the output.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + result = await client._convert_labels_to_ints("account", {"name": "Contoso"}) + assert result["name"] == "Contoso" + + +# --------------------------------------------------------------------------- +# _build_* async methods +# --------------------------------------------------------------------------- + + +class TestBuildMethods: + """Tests for _build_* async methods that produce _RawRequest objects without I/O.""" + + async def test_build_create_post_request(self): + """_build_create() produces a POST request targeting the entity set URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create("accounts", "account", {"amount": 100}) + assert req.method == "POST" + assert "accounts" in req.url + + async def test_build_create_multiple_post_request(self): + """_build_create_multiple() produces a POST targeting the CreateMultiple action.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create_multiple("accounts", "account", [{"amount": 100}]) + assert req.method == "POST" + assert "CreateMultiple" in req.url + + async def test_build_create_multiple_injects_odata_type(self): + """Each entry in the Targets list receives an @odata.type annotation.""" + client = _make_client() + _seed_cache(client) + req = await client._build_create_multiple("accounts", "account", [{"amount": 100}]) + body = json.loads(req.body) + assert "@odata.type" in body["Targets"][0] + + async def test_build_update_patch_request(self): + """_build_update() produces a PATCH request with an If-Match: * concurrency guard.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update("account", "guid-1", {"name": "X"}) + assert req.method == "PATCH" + assert "accounts" in req.url + assert req.headers.get("If-Match") == "*" + + async def test_build_update_with_content_id_reference(self): + """A $-prefixed record_id is used as a raw changeset content-ID reference URL.""" + client = _make_client() + req = await client._build_update("account", "$1", {"name": "X"}) + assert req.url == "$1" + + async def test_build_delete_delete_request(self): + """_build_delete() produces a DELETE request with an If-Match: * concurrency guard.""" + client = _make_client() + _seed_cache(client) + req = await client._build_delete("account", "guid-1") + assert req.method == "DELETE" + assert "accounts" in req.url + assert req.headers.get("If-Match") == "*" + + async def test_build_delete_with_content_id_reference(self): + """A $-prefixed record_id is used as a raw changeset content-ID reference URL.""" + client = _make_client() + req = await client._build_delete("account", "$2") + assert req.url == "$2" + + async def test_build_get_get_request_with_select(self): + """_build_get() encodes the select list as a $select query string parameter.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1", select=["name", "telephone1"]) + assert req.method == "GET" + assert "accounts" in req.url + assert "$select=name,telephone1" in req.url + + async def test_build_get_no_select(self): + """_build_get() omits $select from the URL when no columns are specified.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1") + assert "$select" not in req.url + + async def test_build_sql_encodes_query(self): + """_build_sql() produces a GET request with the SQL statement in a sql= parameter.""" + client = _make_client() + _seed_cache(client) + req = await client._build_sql("SELECT name FROM account") + assert req.method == "GET" + assert "sql=" in req.url + assert "SELECT" in req.url or "SELECT" in req.url.replace("%20", " ") + + async def test_build_upsert_patch_request(self): + """_build_upsert() produces a PATCH without If-Match, allowing create-or-replace semantics.""" + client = _make_client() + req = await client._build_upsert("accounts", "account", {"accountnumber": "A"}, {"name": "X"}) + assert req.method == "PATCH" + assert "accounts" in req.url + assert req.headers is None or "If-Match" not in req.headers + + async def test_build_upsert_multiple_post_request(self): + """_build_upsert_multiple() produces a POST targeting the UpsertMultiple action.""" + client = _make_client() + req = await client._build_upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"name": "X"}], + ) + assert req.method == "POST" + assert "UpsertMultiple" in req.url + + async def test_build_upsert_multiple_mismatched_raises(self): + """ValidationError is raised when the alternate-key and record lists differ in length.""" + client = _make_client() + with pytest.raises(ValidationError): + await client._build_upsert_multiple("accounts", "account", [{"k": "1"}], [{"n": "A"}, {"n": "B"}]) + + async def test_build_upsert_multiple_key_conflict_raises(self): + """ValidationError is raised when a record field overwrites an alternate-key field.""" + client = _make_client() + with pytest.raises(ValidationError, match="conflicts"): + await client._build_upsert_multiple( + "accounts", + "account", + [{"accountnumber": "A"}], + [{"accountnumber": "B"}], + ) + + async def test_build_delete_multiple_bulk_delete(self): + """_build_delete_multiple() produces a POST BulkDelete request with a QuerySet body.""" + client = _make_client() + _seed_cache(client) + req = await client._build_delete_multiple("account", ["id-1", "id-2"]) + assert req.method == "POST" + assert "BulkDelete" in req.url + body = json.loads(req.body) + assert "QuerySet" in body + + async def test_build_update_multiple_broadcast(self): + """A dict for changes is broadcast to all IDs; Targets list length matches ID count.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update_multiple("accounts", "account", ["id-1", "id-2"], {"name": "X"}) + assert req.method == "POST" + assert "UpdateMultiple" in req.url + body = json.loads(req.body) + assert len(body["Targets"]) == 2 + + async def test_build_update_multiple_paired(self): + """A list for changes is applied pairwise; Targets list matches the paired length.""" + client = _make_client() + _seed_cache(client) + req = await client._build_update_multiple("accounts", "account", ["id-1"], [{"name": "X"}]) + assert req.method == "POST" + body = json.loads(req.body) + assert len(body["Targets"]) == 1 + + async def test_build_update_multiple_invalid_changes_type_raises(self): + """ValidationError is raised when changes is neither a dict nor a list.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValidationError): + await client._build_update_multiple("accounts", "account", ["id-1"], "invalid") + + async def test_build_update_multiple_mismatched_length_raises(self): + """ValidationError is raised when the ID list and changes list differ in length.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(ValidationError): + await client._build_update_multiple("accounts", "account", ["id-1", "id-2"], [{"name": "X"}]) + + +# --------------------------------------------------------------------------- +# _wait_for_attribute_visibility() +# --------------------------------------------------------------------------- + + +class TestWaitForAttributeVisibility: + """Tests for _wait_for_attribute_visibility() polling loop.""" + + async def test_succeeds_on_first_attempt(self): + """Returns immediately when the first probe request succeeds.""" + client = _make_client() + client._request.return_value = _resp(status=200) + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0,)) + client._request.assert_called_once() + + async def test_raises_after_all_delays_exhausted(self): + """RuntimeError is raised when every probe attempt fails and delays are exhausted.""" + client = _make_client() + client._request.side_effect = Exception("not visible") + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(RuntimeError, match="did not become visible"): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0, 0)) + + async def test_succeeds_after_retry(self): + """A transient failure on the first probe does not prevent success on the second.""" + client = _make_client() + client._request.side_effect = [Exception("not ready"), _resp(status=200)] + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + await client._wait_for_attribute_visibility("accounts", "new_notes", delays=(0, 0)) + assert client._request.call_count == 2 + + +# --------------------------------------------------------------------------- +# _request_metadata_with_retry() +# --------------------------------------------------------------------------- + + +class TestRequestMetadataWithRetry: + """Tests for _request_metadata_with_retry() which retries on transient 404 responses.""" + + async def test_success_on_first_attempt(self): + """A successful response is returned immediately without any retry.""" + client = _make_client() + client._request.return_value = _resp(status=200, json_data={"value": []}) + result = await client._request_metadata_with_retry("get", "https://example/url") + assert result.status == 200 + + async def test_non_404_raises_immediately(self): + """A non-404 HttpError is re-raised without retrying.""" + client = _make_client() + err = HttpError("Server error", status_code=500) + client._request.side_effect = err + with pytest.raises(HttpError): + await client._request_metadata_with_retry("get", "https://example/url") + assert client._request.call_count == 1 + + async def test_404_retries_and_raises_runtime_error(self): + """A 404 is retried max_attempts=5 times before RuntimeError is raised.""" + client = _make_client() + err = HttpError("Not found", status_code=404) + client._request.side_effect = err + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock): + with pytest.raises(RuntimeError, match="Metadata request failed"): + await client._request_metadata_with_retry("get", "https://example/url") + assert client._request.call_count == 5 # max_attempts defined in implementation + + +# --------------------------------------------------------------------------- +# Additional coverage tests +# --------------------------------------------------------------------------- + + +class TestRequestMergeAndEdgeCases: + """Coverage for _request() header-merge, text-decode failure, and Retry-After edge cases.""" + + def _auth_client(self): + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + return _AsyncODataClient(auth, "https://example.crm.dynamics.com") + + async def test_caller_headers_merged_with_base_headers(self): + """Headers passed by the caller are merged on top of base headers.""" + client = self._auth_client() + client._raw_request = AsyncMock(return_value=_resp(status=200, json_data={})) + await client._request( + "get", "https://example.crm.dynamics.com/api/data/v9.2/accounts", headers={"X-Custom": "value"} + ) + _, kwargs = client._raw_request.call_args + assert kwargs.get("headers", {}).get("X-Custom") == "value" + assert "Authorization" in kwargs.get("headers", {}) + + async def test_non_json_body_still_raises_http_error(self): + """When r.text is non-JSON, _request still raises HttpError with the status code.""" + client = self._auth_client() + r = MagicMock() + r.status = 400 + r.headers = {} + r.text = "not valid json \xff" + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.status_code == 400 + + async def test_retry_after_non_integer_handled(self): + """A non-integer Retry-After header (e.g. HTTP-date) does not crash _request.""" + client = self._auth_client() + body = {"error": {"code": "429", "message": "Too many requests"}} + r = _resp(status=429, json_data=body, headers={"Retry-After": "Wed, 21 Oct 2025 07:28:00 GMT"}) + client._raw_request = AsyncMock(return_value=r) + with pytest.raises(HttpError) as exc: + await client._request("get", "https://example.crm.dynamics.com/api/data/v9.2/accounts") + assert exc.value.to_dict()["details"].get("retry_after") is None + + +class TestCreateMultipleEdgeCases: + """Coverage for _create_multiple() JSON-parse and non-dict body paths.""" + + async def test_json_parse_failure_returns_empty_list(self): + """When response JSON cannot be parsed, returns empty list without raising.""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + async def test_non_dict_body_returns_empty_list(self): + """When response body is a list (not dict), returns empty list.""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data=[1, 2, 3], status=200) + result = await client._create_multiple("accounts", "account", [{"amount": 1}]) + assert result == [] + + +class TestGetMultipleEdgeCases: + """Coverage for _get_multiple() JSON-parse failure path.""" + + async def test_json_parse_failure_returns_empty_page(self): + """When _do_request JSON parse fails, an empty dict is returned (no crash).""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + pages = [page async for page in client._get_multiple("account")] + assert pages == [] + + +class TestQuerySqlEdgeCases: + """Coverage for _query_sql() JSON-parse, non-dict body, and pagination error paths.""" + + async def test_json_parse_failure_returns_empty_list(self): + """When JSON parse fails on first response, returns empty list.""" + client = _make_client() + _seed_cache(client) + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._query_sql("SELECT name FROM account") + assert result == [] + + async def test_non_dict_body_returns_empty_list(self): + """When response body is not a dict (e.g. bare list of non-dicts), returns [].""" + client = _make_client() + _seed_cache(client) + client._request.return_value = _resp(json_data="not-a-dict", status=200) + result = await client._query_sql("SELECT name FROM account") + assert result == [] + + async def test_pagination_duplicate_cookie_warns_and_stops(self): + """Duplicate pagingcookie in $skiptoken triggers RuntimeWarning and stops pagination.""" + import warnings + from urllib.parse import quote + + client = _make_client() + _seed_cache(client) + # Build two skiptokens with the same pagingcookie value but different pagenumbers. + # _extract_pagingcookie extracts the pagingcookie= attribute value; if it's the + # same in both pages, the duplicate-cookie guard fires. + cookie_val = "%3Ccookie+guid%3D%22abc%22%3E" # same encoded cookie in both pages + outer1 = f'' + outer2 = f'' + next_link1 = f"https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken={quote(outer1)}" + next_link2 = f"https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken={quote(outer2)}" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link1, + }, + status=200, + ) + page2 = _resp( + json_data={ + "value": [{"name": "B", "accountid": "g2"}], + "@odata.nextLink": next_link2, + }, + status=200, + ) + client._request.side_effect = [page1, page2] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("pagingcookie" in str(warning.message) for warning in w) + assert len(result) >= 1 + + async def test_pagination_next_page_request_fails_warns_and_stops(self): + """When the next-page request raises, a RuntimeWarning is emitted and pagination stops.""" + import warnings + + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + client._request.side_effect = [page1, HttpError("server error", status_code=500)] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("next-page request failed" in str(warning.message) for warning in w) + assert result == [{"name": "A", "accountid": "g1"}] + + async def test_pagination_next_page_non_json_warns_and_stops(self): + """When the next-page response is not JSON, a RuntimeWarning is emitted.""" + import warnings + + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + bad_resp = MagicMock() + bad_resp.status = 200 + bad_resp.headers = {} + bad_resp.json = MagicMock(side_effect=ValueError("not json")) + client._request.side_effect = [page1, bad_resp] + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await client._query_sql("SELECT name FROM account") + assert any("not valid JSON" in str(warning.message) for warning in w) + assert result == [{"name": "A", "accountid": "g1"}] + + async def test_pagination_non_dict_page_body_stops(self): + """When a paginated response body is not a dict, pagination stops cleanly.""" + client = _make_client() + _seed_cache(client) + next_link = "https://example.crm.dynamics.com/api/data/v9.2/accounts?$skiptoken=abc" + page1 = _resp( + json_data={ + "value": [{"name": "A", "accountid": "g1"}], + "@odata.nextLink": next_link, + }, + status=200, + ) + page2 = _resp(json_data="not-a-dict", status=200) + client._request.side_effect = [page1, page2] + result = await client._query_sql("SELECT name FROM account") + assert result == [{"name": "A", "accountid": "g1"}] + + +class TestPrimaryIdAttrEdgeCases: + """Coverage for _primary_id_attr() RuntimeError when metadata lacks PrimaryIdAttribute.""" + + async def test_raises_when_pk_not_in_cache_after_metadata_fetch(self): + """RuntimeError raised when entity resolves but PrimaryIdAttribute is absent from cache.""" + client = _make_client() + # Populate entity set cache but NOT the primaryid cache + key = client._normalize_cache_key("account") + client._logical_to_entityset_cache[key] = "accounts" + # _entity_set_from_schema_name will hit the cache and return without populating primaryid + with pytest.raises(RuntimeError, match="PrimaryIdAttribute not resolved"): + await client._primary_id_attr("account") + + +class TestGetAttributeMetadataEdgeCases: + """Coverage for _get_attribute_metadata() skip and JSON-parse-failure paths.""" + + async def test_skips_at_sign_fields_in_extra_select(self): + """Fields starting with '@' in extra_select are silently ignored.""" + client = _make_client() + client._request.return_value = _resp( + json_data={"value": [{"MetadataId": "m1", "LogicalName": "name", "SchemaName": "Name"}]}, + status=200, + ) + result = await client._get_attribute_metadata("meta-1", "name", extra_select="@odata.type,AttributeType") + assert result is not None + assert result["LogicalName"] == "name" + + async def test_json_parse_failure_returns_none(self): + """When response JSON parse fails, None is returned without raising.""" + client = _make_client() + r = MagicMock() + r.status = 200 + r.headers = {} + r.json = MagicMock(side_effect=ValueError("not json")) + client._request.return_value = r + result = await client._get_attribute_metadata("meta-1", "name") + assert result is None + + +class TestPicklistEdgeCases: + """Coverage for _bulk_fetch_picklists() guard clauses and _convert_labels_to_ints() paths.""" + + async def test_bulk_fetch_cache_hit_inside_lock_skips_fetch(self): + """Second TTL check inside the lock exits early when another coroutine populated cache.""" + import time + + client = _make_client() + key = client._normalize_cache_key("account") + # Pre-populate cache with a fresh entry (TTL not expired) + client._picklist_label_cache[key] = {"ts": time.time(), "picklists": {}} + # Warm the outer cache check too + client._picklist_label_cache[key]["ts"] = time.time() + # Should return without calling _request + await client._bulk_fetch_picklists("account") + client._request_metadata_with_retry = AsyncMock() + client._request_metadata_with_retry.assert_not_called() + + async def test_bulk_fetch_skips_non_dict_items(self): + """Non-dict items in the picklist response value list are skipped.""" + client = _make_client() + r = _resp(json_data={"value": ["not-a-dict", {"LogicalName": "status", "OptionSet": {"Options": []}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_empty_logical_name(self): + """Items with empty LogicalName are skipped during picklist fetch.""" + client = _make_client() + r = _resp(json_data={"value": [{"LogicalName": "", "OptionSet": {"Options": []}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_non_dict_options(self): + """Non-dict entries in OptionSet.Options are skipped.""" + client = _make_client() + r = _resp(json_data={"value": [{"LogicalName": "status", "OptionSet": {"Options": ["bad"]}}]}) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_bulk_fetch_skips_non_int_value(self): + """Options whose Value is not an int are skipped.""" + client = _make_client() + r = _resp( + json_data={ + "value": [ + { + "LogicalName": "status", + "OptionSet": {"Options": [{"Value": "not-an-int", "Label": {}}]}, + } + ] + } + ) + client._request_metadata_with_retry = AsyncMock(return_value=r) + await client._bulk_fetch_picklists("account") # should not raise + + async def test_convert_labels_non_dict_cache_entry_returns_record(self): + """When picklist cache entry is not a dict, record is returned unchanged.""" + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = "not-a-dict" + result = await client._convert_labels_to_ints("account", {"status": "Active"}) + assert result == {"status": "Active"} + + async def test_convert_labels_skips_odata_annotation_keys(self): + """Keys containing '@odata.' are not looked up in the picklist cache.""" + import time + + client = _make_client() + key = client._normalize_cache_key("account") + client._picklist_label_cache[key] = { + "ts": time.time(), + "picklists": {"status": {"active": 1}}, + } + record = {"status": "active", "status@odata.type": "#Microsoft.Dynamics.CRM.StatusType"} + result = await client._convert_labels_to_ints("account", record) + # status resolved; odata annotation key left untouched + assert result["status"] == 1 + assert "status@odata.type" in result + + +class TestCreateEntityEdgeCases: + """Coverage for _create_entity() solution_name, missing EntitySetName, missing MetadataId.""" + + async def test_create_entity_with_solution_unique_name(self): + """solution_unique_name is passed as a query parameter to the POST request.""" + client = _make_client() + client._request.return_value = _resp(status=204) + entity_resp = { + "LogicalName": "new_table", + "EntitySetName": "new_tables", + "MetadataId": "meta-999", + "SchemaName": "new_table", + "PrimaryIdAttribute": "new_tableid", + } + client._get_entity_by_table_schema_name = AsyncMock(return_value=entity_resp) + result = await client._create_entity( + "new_table", + "New Table", + [], + solution_unique_name="MySolution", + ) + _, kwargs = client._request.call_args + assert kwargs.get("params", {}).get("SolutionUniqueName") == "MySolution" + assert result["EntitySetName"] == "new_tables" + + async def test_create_entity_missing_entity_set_name_raises(self): + """RuntimeError raised when EntitySetName is absent after create.""" + client = _make_client() + client._request.return_value = _resp(status=204) + client._get_entity_by_table_schema_name = AsyncMock(return_value={"MetadataId": "m1"}) + with pytest.raises(RuntimeError, match="EntitySetName not available"): + await client._create_entity("t", "t", "T", []) + + async def test_create_entity_missing_metadata_id_raises(self): + """RuntimeError raised when MetadataId is absent after create.""" + client = _make_client() + client._request.return_value = _resp(status=204) + client._get_entity_by_table_schema_name = AsyncMock(return_value={"EntitySetName": "ts", "LogicalName": "t"}) + with pytest.raises(RuntimeError, match="MetadataId missing"): + await client._create_entity("t", "t", "T", []) + + +class TestWaitForAttributeVisibilityWithDelay: + """Coverage for _wait_for_attribute_visibility() sleep branch.""" + + async def test_waits_when_delay_is_nonzero(self): + """asyncio.sleep is called when the computed delay is positive.""" + client = _make_client() + # First call (delay=0) fails so the loop continues to delay=1 where sleep fires. + ok = _resp(status=200) + client._request.side_effect = [HttpError("not yet", status_code=404), ok] + with patch("PowerPlatform.Dataverse.aio.data._async_odata.asyncio.sleep", new_callable=AsyncMock) as mock_sleep: + await client._wait_for_attribute_visibility("accounts", "new_col", delays=(0, 1)) + mock_sleep.assert_called_once_with(1) + + +class TestAlternateKeyWithDisplayName: + """Coverage for _create_alternate_key() display_name_label path.""" + + async def test_create_alternate_key_with_display_name(self): + """DisplayName payload key is set when display_name_label is provided.""" + client = _make_client() + ent = {"LogicalName": "account", "EntitySetName": "accounts", "MetadataId": "m1", "SchemaName": "Account"} + client._get_entity_by_table_schema_name = AsyncMock(return_value=ent) + r = _resp(status=204, headers={"OData-EntityId": "https://example.com/(key123)"}) + r.headers = {"OData-EntityId": "https://example.com/(key123)"} + client._request.return_value = r + + label = MagicMock() + label.to_dict.return_value = {"UserLocalizedLabel": {"Label": "Account Number", "LanguageCode": 1033}} + + result = await client._create_alternate_key("account", "AccountNumber_AK", ["accountnumber"], label) + assert result["schema_name"] == "AccountNumber_AK" + _, kwargs = client._request.call_args + assert "DisplayName" in kwargs.get("json", {}) + + +class TestBuildMethodsAdditional: + """Coverage for _build_create_multiple TypeError, _build_get annotations, and _build_list.""" + + async def test_build_create_multiple_non_dict_raises_type_error(self): + """_build_create_multiple() raises TypeError when records contain non-dicts.""" + client = _make_client() + _seed_cache(client) + with pytest.raises(TypeError, match="dicts"): + await client._build_create_multiple("accounts", "account", ["not-a-dict"]) + + async def test_build_get_with_include_annotations(self): + """_build_get() sets Prefer header when include_annotations is specified.""" + client = _make_client() + _seed_cache(client) + req = await client._build_get("account", "guid-1", include_annotations="*") + assert req.headers is not None + assert "odata.include-annotations" in req.headers.get("Prefer", "") + + async def test_build_list_basic(self): + """_build_list() produces a GET request targeting the entity-set URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account") + assert req.method == "GET" + assert "accounts" in req.url + assert req.headers is None + + async def test_build_list_with_select_filter_orderby_top(self): + """_build_list() encodes all OData query parameters into the URL.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list( + "account", + select=["name", "telephone1"], + filter="statecode eq 0", + orderby=["name asc"], + top=10, + ) + assert "$select=name,telephone1" in req.url + assert "$filter=statecode+eq+0" in req.url or "$filter=statecode%20eq%200" in req.url or "statecode" in req.url + assert "$top=10" in req.url + + async def test_build_list_with_page_size_and_annotations(self): + """_build_list() sets Prefer header for page_size and include_annotations.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account", page_size=50, include_annotations="*") + assert req.headers is not None + prefer = req.headers.get("Prefer", "") + assert "odata.maxpagesize=50" in prefer + assert "odata.include-annotations" in prefer + + async def test_build_list_with_count(self): + """_build_list() appends $count=true when count=True.""" + client = _make_client() + _seed_cache(client) + req = await client._build_list("account", count=True) + assert "$count=true" in req.url + + +class TestAsyncOperationContextUserAgent: + """User-Agent header reflects operation_context on the async client.""" + + async def test_default_user_agent_unchanged(self): + from PowerPlatform.Dataverse.aio.data._async_odata import _USER_AGENT + + client = _make_client() + headers = await client._headers() + assert headers["User-Agent"] == _USER_AGENT + + async def test_operation_context_appended(self): + from PowerPlatform.Dataverse.aio.data._async_odata import _USER_AGENT + from PowerPlatform.Dataverse.core.config import DataverseConfig, OperationContext + + ctx_str = "app=dataverse-skills/1.2.1;agent=claude-code" + config = DataverseConfig(operation_context=OperationContext(user_agent_context=ctx_str)) + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com", config=config) + headers = await client._headers() + assert headers["User-Agent"] == f"{_USER_AGENT} ({ctx_str})" + + async def test_none_context_no_parentheses(self): + from PowerPlatform.Dataverse.core.config import DataverseConfig + + config = DataverseConfig(operation_context=None) + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="test-token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com", config=config) + headers = await client._headers() + assert "(" not in headers["User-Agent"] diff --git a/tests/unit/aio/data/test_async_relationships.py b/tests/unit/aio/data/test_async_relationships.py new file mode 100644 index 00000000..cf0ed1bd --- /dev/null +++ b/tests/unit/aio/data/test_async_relationships.py @@ -0,0 +1,314 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncRelationshipOperationsMixin.""" + +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient +from PowerPlatform.Dataverse.core.errors import MetadataError +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + ManyToManyRelationshipMetadata, + OneToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(json_data=None, status=200, headers=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.json = MagicMock(return_value=json_data if json_data is not None else {}) + return r + + +def _entity_def(meta_id="meta-001", logical="account"): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": "accounts", + "PrimaryIdAttribute": "accountid", + "MetadataId": meta_id, + "SchemaName": "Account", + } + ] + } + + +def _label(text: str = "Test") -> Label: + """Return a Label with a single English localized label.""" + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +def _seed_cache(client, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +# --------------------------------------------------------------------------- +# _extract_id_from_header (sync) +# --------------------------------------------------------------------------- + + +class TestExtractIdFromHeader: + """Tests for _extract_id_from_header(), which parses GUIDs from OData-EntityId URLs. + + The regex matches only hex characters and dashes inside parentheses, so + only proper UUID-format strings are extracted. + """ + + def test_extracts_guid_from_url(self): + """A UUID enclosed in parentheses at the end of a URL is returned.""" + client = _make_client() + guid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + header = f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + result = client._extract_id_from_header(header) + assert result == guid + + def test_returns_none_for_empty_header(self): + """None is returned for both None and empty-string inputs.""" + client = _make_client() + assert client._extract_id_from_header(None) is None + assert client._extract_id_from_header("") is None + + def test_returns_none_when_no_guid(self): + """None is returned when the header contains no hex UUID in parentheses.""" + client = _make_client() + assert client._extract_id_from_header("no-guid-here") is None + + +# --------------------------------------------------------------------------- +# _create_one_to_many_relationship() +# --------------------------------------------------------------------------- + + +class TestCreateOneToManyRelationship: + """Tests for _create_one_to_many_relationship() one-to-many relationship creation.""" + + async def test_success(self): + """The relationship ID, schema name, and lookup schema name are returned on success.""" + client = _make_client() + guid = "a1b2c3d4-e5f6-7890-abcd-ef1234567890" + client._request.return_value = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + }, + ) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Dept")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + result = await client._create_one_to_many_relationship(lookup, relationship) + assert result["relationship_id"] == guid + assert result["relationship_schema_name"] == "new_Dept_Emp" + assert result["lookup_schema_name"] == "new_DeptId" + + async def test_with_solution(self): + """The MSCRM.SolutionUniqueName header is injected when a solution name is supplied.""" + client = _make_client() + client._request.return_value = _resp(status=204, headers={}) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Dept")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + await client._create_one_to_many_relationship(lookup, relationship, solution="MySolution") + call_kwargs = client._request.call_args.kwargs + headers = call_kwargs.get("headers", {}) + assert "MSCRM.SolutionUniqueName" in headers + assert headers["MSCRM.SolutionUniqueName"] == "MySolution" + + +# --------------------------------------------------------------------------- +# _create_many_to_many_relationship() +# --------------------------------------------------------------------------- + + +class TestCreateManyToManyRelationship: + """Tests for _create_many_to_many_relationship() many-to-many relationship creation.""" + + async def test_success(self): + """The relationship ID and entity names are returned on success.""" + client = _make_client() + guid = "b2c3d4e5-f6a7-8901-bcde-f12345678901" + client._request.return_value = _resp( + status=204, + headers={ + "OData-EntityId": f"https://example.crm.dynamics.com/api/data/v9.2/RelationshipDefinitions({guid})" + }, + ) + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + result = await client._create_many_to_many_relationship(relationship) + assert result["relationship_id"] == guid + assert result["relationship_schema_name"] == "new_emp_proj" + assert result["entity1_logical_name"] == "new_employee" + assert result["entity2_logical_name"] == "new_project" + + async def test_with_solution(self): + """The MSCRM.SolutionUniqueName header is injected when a solution name is supplied.""" + client = _make_client() + client._request.return_value = _resp(status=204, headers={}) + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + await client._create_many_to_many_relationship(relationship, solution="MySol") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("MSCRM.SolutionUniqueName") == "MySol" + + +# --------------------------------------------------------------------------- +# _delete_relationship() +# --------------------------------------------------------------------------- + + +class TestDeleteRelationship: + """Tests for _delete_relationship() relationship removal by GUID.""" + + async def test_issues_delete(self): + """A DELETE request is issued containing the relationship GUID in the URL.""" + client = _make_client() + client._request.return_value = _resp(status=204) + await client._delete_relationship("rel-guid-1") + call_args = client._request.call_args + assert call_args.args[0] == "delete" + assert "rel-guid-1" in call_args.args[1] + + async def test_sets_if_match_header(self): + """An If-Match: * header is sent to prevent accidental deletion of a stale version.""" + client = _make_client() + client._request.return_value = _resp(status=204) + await client._delete_relationship("rel-guid-1") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("If-Match") == "*" + + +# --------------------------------------------------------------------------- +# _get_relationship() +# --------------------------------------------------------------------------- + + +class TestGetRelationship: + """Tests for _get_relationship() single-relationship lookup by schema name.""" + + async def test_returns_relationship_dict(self): + """The first matching relationship dict from the value list is returned.""" + client = _make_client() + rel = {"SchemaName": "new_Dept_Emp", "RelationshipId": "rel-1"} + client._request.return_value = _resp(json_data={"value": [rel]}) + result = await client._get_relationship("new_Dept_Emp") + assert result == rel + + async def test_returns_none_when_not_found(self): + """None is returned when the API returns an empty value list.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + result = await client._get_relationship("nonexistent") + assert result is None + + +# --------------------------------------------------------------------------- +# _list_relationships() +# --------------------------------------------------------------------------- + + +class TestListRelationships: + """Tests for _list_relationships() global relationship listing.""" + + async def test_returns_all_relationships(self): + """The full value list is returned when no filter is applied.""" + client = _make_client() + rels = [{"SchemaName": "rel1"}, {"SchemaName": "rel2"}] + client._request.return_value = _resp(json_data={"value": rels}) + result = await client._list_relationships() + assert result == rels + + async def test_with_filter_and_select(self): + """Optional filter and select parameters are forwarded as OData query params.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + result = await client._list_relationships( + filter="RelationshipType eq 'OneToMany'", + select=["SchemaName"], + ) + assert result == [] + call_kwargs = client._request.call_args.kwargs + params = call_kwargs.get("params", {}) + assert "$filter" in params + assert "$select" in params + + +# --------------------------------------------------------------------------- +# _list_table_relationships() +# --------------------------------------------------------------------------- + + +class TestListTableRelationships: + """Tests for _list_table_relationships() which aggregates all three relationship types.""" + + async def test_combines_three_relationship_types(self): + """One-to-many, many-to-one, and many-to-many relationships are combined into one list.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def()) + otm_resp = _resp(json_data={"value": [{"SchemaName": "rel_otm"}]}) + mto_resp = _resp(json_data={"value": [{"SchemaName": "rel_mto"}]}) + mtm_resp = _resp(json_data={"value": [{"SchemaName": "rel_mtm"}]}) + client._request.side_effect = [entity_resp, otm_resp, mto_resp, mtm_resp] + result = await client._list_table_relationships("account") + assert len(result) == 3 + schema_names = [r["SchemaName"] for r in result] + assert "rel_otm" in schema_names + assert "rel_mto" in schema_names + assert "rel_mtm" in schema_names + + async def test_table_not_found_raises(self): + """MetadataError is raised when the table does not exist in entity metadata.""" + client = _make_client() + client._request.return_value = _resp(json_data={"value": []}) + with pytest.raises(MetadataError, match="not found"): + await client._list_table_relationships("nonexistent") + + async def test_with_filter_and_select(self): + """Optional filter and select parameters are forwarded to all three relationship requests.""" + client = _make_client() + entity_resp = _resp(json_data=_entity_def()) + empty_resp = _resp(json_data={"value": []}) + client._request.side_effect = [entity_resp, empty_resp, empty_resp, empty_resp] + result = await client._list_table_relationships( + "account", + filter="IsCustomRelationship eq true", + select=["SchemaName"], + ) + assert result == [] diff --git a/tests/unit/aio/data/test_async_upload.py b/tests/unit/aio/data/test_async_upload.py new file mode 100644 index 00000000..d5268913 --- /dev/null +++ b/tests/unit/aio/data/test_async_upload.py @@ -0,0 +1,318 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for _AsyncFileUploadMixin.""" + +import os +import tempfile +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client() -> _AsyncODataClient: + """Return _AsyncODataClient with _request mocked at the HTTP boundary.""" + auth = MagicMock() + auth._acquire_token = AsyncMock(return_value=MagicMock(access_token="token")) + client = _AsyncODataClient(auth, "https://example.crm.dynamics.com") + client._request = AsyncMock() + return client + + +def _resp(status=200, headers=None, json_data=None): + """Create a mock _AsyncResponse-compatible response.""" + r = MagicMock() + r.status = status + r.headers = headers or {} + r.json = MagicMock(return_value=json_data or {}) + return r + + +def _seed_cache(client, table="account", entity_set="accounts", pk="accountid"): + """Pre-populate entity-set and primary-ID caches to bypass HTTP for schema-name lookups.""" + key = client._normalize_cache_key(table) + client._logical_to_entityset_cache[key] = entity_set + client._logical_primaryid_cache[key] = pk + + +def _entity_def(meta_id="meta-001", entity_set="accounts", logical="account"): + """Return a minimal EntityDefinitions value-list response body.""" + return { + "value": [ + { + "LogicalName": logical, + "EntitySetName": entity_set, + "PrimaryIdAttribute": "accountid", + "MetadataId": meta_id, + "SchemaName": "Account", + } + ] + } + + +# --------------------------------------------------------------------------- +# _upload_file_small() +# --------------------------------------------------------------------------- + + +class TestUploadFileSmall: + """Tests for _upload_file_small(), the single-request upload path for small files.""" + + async def test_success_uploads_with_patch(self): + """A successful upload issues a PATCH with x-ms-file-name and If-None-Match headers.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path) + call_args = client._request.call_args + assert call_args.args[0] == "patch" + headers = call_args.kwargs.get("headers", {}) + assert "x-ms-file-name" in headers + assert headers.get("If-None-Match") == "null" + finally: + os.unlink(path) + + async def test_success_with_overwrite(self): + """When if_none_match=False, an If-Match: * header is sent instead of If-None-Match.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path, if_none_match=False) + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("If-Match") == "*" + assert "If-None-Match" not in headers + finally: + os.unlink(path) + + async def test_explicit_mime_type(self): + """An explicit content_type is forwarded as the Content-Type header.""" + client = _make_client() + client._request.return_value = _resp(status=204) + with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as f: + f.write(b"%PDF") + path = f.name + try: + await client._upload_file_small("accounts", "guid-1", "new_document", path, content_type="application/pdf") + headers = client._request.call_args.kwargs.get("headers", {}) + assert headers.get("Content-Type") == "application/pdf" + finally: + os.unlink(path) + + async def test_empty_record_id_raises(self): + """ValueError is raised immediately when record_id is an empty string.""" + client = _make_client() + with pytest.raises(ValueError, match="record_id required"): + await client._upload_file_small("accounts", "", "new_doc", "/any/path") + + async def test_file_not_found_raises(self): + """FileNotFoundError is raised when the specified file path does not exist.""" + client = _make_client() + with pytest.raises(FileNotFoundError): + await client._upload_file_small("accounts", "guid-1", "new_doc", "/nonexistent/path.txt") + + async def test_file_too_large_raises(self): + """ValueError is raised when the file size exceeds the single-upload size limit.""" + client = _make_client() + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"x") + path = f.name + try: + stat_result = MagicMock() + stat_result.st_size = 200 * 1024 * 1024 + stat_result.st_mode = 0o100644 # regular file + with patch("pathlib.Path.stat", return_value=stat_result): + with pytest.raises(ValueError, match="exceeds single-upload limit"): + await client._upload_file_small("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# _upload_file_chunk() +# --------------------------------------------------------------------------- + + +class TestUploadFileChunk: + """Tests for _upload_file_chunk(), the chunked upload path for large files.""" + + async def test_success_single_chunk(self): + """A small file completes in two requests: session init and one chunk PUT.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_document?sessiontoken=xyz" + init_resp = _resp(status=200, headers={"Location": location, "x-ms-chunk-size": "4194304"}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello world") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_document", path) + assert client._request.call_count == 2 + finally: + os.unlink(path) + + async def test_success_with_if_match(self): + """When if_none_match=False, an If-Match: * header is included in the session-init request.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_document?sessiontoken=abc" + init_resp = _resp(status=200, headers={"Location": location}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_document", path, if_none_match=False) + init_headers = client._request.call_args_list[0].kwargs.get("headers", {}) + assert init_headers.get("If-Match") == "*" + finally: + os.unlink(path) + + async def test_empty_record_id_raises(self): + """ValueError is raised immediately when record_id is an empty string.""" + client = _make_client() + with pytest.raises(ValueError, match="record_id required"): + await client._upload_file_chunk("accounts", "", "new_doc", "/any/path") + + async def test_file_not_found_raises(self): + """FileNotFoundError is raised when the specified file path does not exist.""" + client = _make_client() + with pytest.raises(FileNotFoundError): + await client._upload_file_chunk("accounts", "guid-1", "new_doc", "/nonexistent/path.txt") + + async def test_missing_location_header_raises(self): + """RuntimeError is raised when the session-init response lacks a Location header.""" + client = _make_client() + client._request.return_value = _resp(status=200, headers={}) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + with pytest.raises(RuntimeError, match="Missing Location header"): + await client._upload_file_chunk("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + async def test_invalid_chunk_size_falls_back_to_default(self): + """A non-integer x-ms-chunk-size header is ignored and the 4MB default is used.""" + client = _make_client() + location = "https://example.crm.dynamics.com/api/data/v9.2/accounts(guid-1)/new_doc?tok=x" + init_resp = _resp(status=200, headers={"Location": location, "x-ms-chunk-size": "invalid"}) + chunk_resp = _resp(status=204) + client._request.side_effect = [init_resp, chunk_resp] + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"hello") + path = f.name + try: + await client._upload_file_chunk("accounts", "guid-1", "new_doc", path) + finally: + os.unlink(path) + + +# --------------------------------------------------------------------------- +# _upload_file() — auto mode dispatch +# --------------------------------------------------------------------------- + + +class TestUploadFile: + """Tests for _upload_file(), the high-level dispatcher that selects the upload path.""" + + async def test_small_file_uses_small_mode(self): + """mode='small' routes to _upload_file_small without calling _upload_file_chunk.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + client._upload_file_small = AsyncMock(return_value=None) + client._upload_file_chunk = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"small content") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="small") + client._upload_file_small.assert_called_once() + client._upload_file_chunk.assert_not_called() + finally: + os.unlink(path) + + async def test_chunk_mode_uses_chunk_upload(self): + """mode='chunk' routes to _upload_file_chunk without calling _upload_file_small.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + client._upload_file_small = AsyncMock(return_value=None) + client._upload_file_chunk = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".bin") as f: + f.write(b"big content") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="chunk") + client._upload_file_chunk.assert_called_once() + finally: + os.unlink(path) + + async def test_invalid_mode_raises(self): + """ValueError is raised when an unrecognised mode string is supplied.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + with pytest.raises(ValueError, match="Invalid mode"): + await client._upload_file("account", "guid-1", "new_doc", path, mode="badmode") + finally: + os.unlink(path) + + async def test_auto_mode_file_not_found_raises(self): + """FileNotFoundError is raised in default auto mode when the file path does not exist.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value={"MetadataId": "attr-1"}) + with pytest.raises(FileNotFoundError): + await client._upload_file("account", "guid-1", "new_doc", "/nonexistent/file.txt") + + async def test_attribute_not_found_creates_it(self): + """When attribute metadata is missing, _create_columns and _wait_for_attribute_visibility are called.""" + client = _make_client() + _seed_cache(client) + client._get_entity_by_table_schema_name = AsyncMock( + return_value={"MetadataId": "meta-1", "LogicalName": "account"} + ) + client._get_attribute_metadata = AsyncMock(return_value=None) + client._create_columns = AsyncMock(return_value=["new_doc"]) + client._wait_for_attribute_visibility = AsyncMock(return_value=None) + client._upload_file_small = AsyncMock(return_value=None) + with tempfile.NamedTemporaryFile(delete=False, suffix=".txt") as f: + f.write(b"data") + path = f.name + try: + await client._upload_file("account", "guid-1", "new_doc", path, mode="small") + client._create_columns.assert_called_once_with("account", {"new_doc": "file"}) + client._wait_for_attribute_visibility.assert_called_once() + finally: + os.unlink(path) diff --git a/tests/unit/aio/test_async_batch.py b/tests/unit/aio/test_async_batch.py new file mode 100644 index 00000000..baad3ee6 --- /dev/null +++ b/tests/unit/aio/test_async_batch.py @@ -0,0 +1,511 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +import pandas as pd +from unittest.mock import AsyncMock + + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_batch import ( + AsyncBatchOperations, + AsyncBatchRequest, + AsyncChangeSet, +) +from PowerPlatform.Dataverse.operations.batch import ( + BatchRecordOperations, + BatchTableOperations, + BatchQueryOperations, + BatchDataFrameOperations, + ChangeSetRecordOperations, +) +from PowerPlatform.Dataverse.data._batch_base import ( + _RecordCreate, + _RecordUpdate, + _RecordDelete, + _RecordGet, + _RecordUpsert, + _TableCreate, + _TableDelete, + _TableGet, + _TableList, + _TableAddColumns, + _TableRemoveColumns, + _TableCreateOneToMany, + _TableCreateManyToMany, + _TableDeleteRelationship, + _TableGetRelationship, + _TableCreateLookupField, + _QuerySql, + _ChangeSet, +) +from PowerPlatform.Dataverse.models.upsert import UpsertItem +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + + +def _label(text: str = "Test") -> Label: + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +from PowerPlatform.Dataverse.core.errors import ValidationError + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_batch(async_client: AsyncDataverseClient) -> AsyncBatchRequest: + return async_client.batch.new() + + +# --------------------------------------------------------------------------- +# Namespace tests +# --------------------------------------------------------------------------- + + +class TestAsyncBatchOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.batch, AsyncBatchOperations) + + def test_new_returns_batch_request(self, async_client): + batch = async_client.batch.new() + assert isinstance(batch, AsyncBatchRequest) + + def test_batch_request_namespaces(self, async_client): + batch = async_client.batch.new() + assert isinstance(batch.records, BatchRecordOperations) + assert isinstance(batch.tables, BatchTableOperations) + assert isinstance(batch.query, BatchQueryOperations) + assert isinstance(batch.dataframe, BatchDataFrameOperations) + + +# --------------------------------------------------------------------------- +# AsyncBatchRecordOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchRecordOperations: + def test_create_single_appends_record_create(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", {"name": "Contoso"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordCreate) + assert item.table == "account" + assert item.data == {"name": "Contoso"} + + def test_create_bulk_appends_record_create(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", [{"name": "A"}, {"name": "B"}]) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordCreate) + + def test_update_single_appends_record_update(self, async_client): + batch = _make_batch(async_client) + batch.records.update("account", "guid-1", {"name": "X"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordUpdate) + assert item.table == "account" + + def test_delete_single_appends_record_delete(self, async_client): + batch = _make_batch(async_client) + batch.records.delete("account", "guid-1") + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordDelete) + + def test_delete_bulk_appends_record_delete(self, async_client): + batch = _make_batch(async_client) + batch.records.delete("account", ["guid-1", "guid-2"]) + assert isinstance(batch._items[0], _RecordDelete) + + def test_get_appends_record_get(self, async_client): + batch = _make_batch(async_client) + with pytest.warns(DeprecationWarning): + batch.records.get("account", "guid-1", select=["name"]) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _RecordGet) + assert item.table == "account" + assert item.record_id == "guid-1" + assert item.select == ["name"] + + def test_upsert_appends_record_upsert(self, async_client): + batch = _make_batch(async_client) + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "X"}) + batch.records.upsert("account", [item]) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordUpsert) + + def test_upsert_dict_item_normalized(self, async_client): + batch = _make_batch(async_client) + batch.records.upsert("account", [{"alternate_key": {"accountnumber": "ACC-001"}, "record": {"name": "X"}}]) + enqueued = batch._items[0] + assert isinstance(enqueued, _RecordUpsert) + assert isinstance(enqueued.items[0], UpsertItem) + + def test_upsert_empty_list_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.records.upsert("account", []) + + def test_upsert_invalid_item_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.records.upsert("account", [42]) + + +# --------------------------------------------------------------------------- +# AsyncBatchTableOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchTableOperations: + def test_create_appends_table_create(self, async_client): + batch = _make_batch(async_client) + batch.tables.create("new_Product", {"new_Price": "decimal"}) + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _TableCreate) + assert item.table == "new_Product" + + def test_delete_appends_table_delete(self, async_client): + batch = _make_batch(async_client) + batch.tables.delete("new_Product") + assert isinstance(batch._items[0], _TableDelete) + + def test_get_appends_table_get(self, async_client): + batch = _make_batch(async_client) + batch.tables.get("new_Product") + assert isinstance(batch._items[0], _TableGet) + + def test_list_appends_table_list(self, async_client): + batch = _make_batch(async_client) + batch.tables.list() + assert isinstance(batch._items[0], _TableList) + + def test_add_columns_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.add_columns("new_Product", {"new_Notes": "string"}) + assert isinstance(batch._items[0], _TableAddColumns) + + def test_remove_columns_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.remove_columns("new_Product", "new_Notes") + assert isinstance(batch._items[0], _TableRemoveColumns) + + def test_create_one_to_many_appends(self, async_client): + batch = _make_batch(async_client) + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Department")) + rel = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_emp", + referenced_attribute="new_deptid", + ) + batch.tables.create_one_to_many_relationship(lookup, rel) + assert isinstance(batch._items[0], _TableCreateOneToMany) + + def test_create_many_to_many_appends(self, async_client): + batch = _make_batch(async_client) + rel = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_emp", + entity2_logical_name="new_proj", + ) + batch.tables.create_many_to_many_relationship(rel) + assert isinstance(batch._items[0], _TableCreateManyToMany) + + def test_delete_relationship_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.delete_relationship("rel-guid") + assert isinstance(batch._items[0], _TableDeleteRelationship) + + def test_get_relationship_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.get_relationship("new_Dept_Emp") + assert isinstance(batch._items[0], _TableGetRelationship) + + def test_create_lookup_field_appends(self, async_client): + batch = _make_batch(async_client) + batch.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + ) + assert isinstance(batch._items[0], _TableCreateLookupField) + + +# --------------------------------------------------------------------------- +# AsyncBatchQueryOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchQueryOperations: + def test_sql_appends_query_sql(self, async_client): + batch = _make_batch(async_client) + batch.query.sql("SELECT name FROM account") + assert len(batch._items) == 1 + item = batch._items[0] + assert isinstance(item, _QuerySql) + assert item.sql == "SELECT name FROM account" + + def test_sql_strips_whitespace(self, async_client): + batch = _make_batch(async_client) + batch.query.sql(" SELECT name FROM account ") + assert batch._items[0].sql == "SELECT name FROM account" + + def test_sql_empty_string_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql("") + + def test_sql_whitespace_only_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql(" ") + + def test_sql_non_string_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValidationError): + batch.query.sql(None) + + +# --------------------------------------------------------------------------- +# AsyncBatchDataFrameOperations +# --------------------------------------------------------------------------- + + +class TestAsyncBatchDataFrameOperations: + def test_create_from_dataframe(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + batch.dataframe.create("account", df) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordCreate) + + def test_create_non_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.create("account", [{"name": "X"}]) + + def test_create_empty_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.create("account", pd.DataFrame()) + + def test_create_all_null_row_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.create("account", pd.DataFrame([{"name": None}])) + + def test_update_from_dataframe(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1", "name": "X"}]) + batch.dataframe.update("account", df, id_column="accountid") + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordUpdate) + + def test_update_non_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.update("account", [{}], id_column="id") + + def test_update_empty_dataframe_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(ValueError): + batch.dataframe.update("account", pd.DataFrame(), id_column="id") + + def test_update_missing_id_column_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"name": "X"}]) + with pytest.raises(ValueError, match="id_column"): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_invalid_ids_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": None, "name": "X"}]) + with pytest.raises(ValueError): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_no_change_columns_raises(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1"}]) + with pytest.raises(ValueError): + batch.dataframe.update("account", df, id_column="accountid") + + def test_update_all_null_rows_skipped(self, async_client): + batch = _make_batch(async_client) + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": None}]) + batch.dataframe.update("account", df, id_column="accountid") + # All change values null -> nothing enqueued + assert len(batch._items) == 0 + + def test_delete_from_series(self, async_client): + batch = _make_batch(async_client) + ids = pd.Series(["guid-1", "guid-2"]) + batch.dataframe.delete("account", ids) + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _RecordDelete) + + def test_delete_non_series_raises(self, async_client): + batch = _make_batch(async_client) + with pytest.raises(TypeError): + batch.dataframe.delete("account", ["guid-1"]) + + def test_delete_empty_series_no_item(self, async_client): + batch = _make_batch(async_client) + batch.dataframe.delete("account", pd.Series([], dtype=str)) + assert len(batch._items) == 0 + + def test_delete_invalid_ids_raises(self, async_client): + batch = _make_batch(async_client) + ids = pd.Series(["guid-1", None]) + with pytest.raises(ValueError): + batch.dataframe.delete("account", ids) + + +# --------------------------------------------------------------------------- +# AsyncChangeSet +# --------------------------------------------------------------------------- + + +class TestAsyncChangeSet: + def test_changeset_returns_async_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + assert isinstance(cs, AsyncChangeSet) + + def test_changeset_records_namespace(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + assert isinstance(cs.records, ChangeSetRecordOperations) + + def test_changeset_appended_to_items(self, async_client): + batch = _make_batch(async_client) + batch.changeset() + assert len(batch._items) == 1 + assert isinstance(batch._items[0], _ChangeSet) + + async def test_changeset_async_context_manager(self, async_client): + batch = _make_batch(async_client) + async with batch.changeset() as cs: + assert isinstance(cs, AsyncChangeSet) + + +class TestAsyncChangeSetRecordOperations: + def test_create_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + ref = cs.records.create("account", {"name": "Contoso"}) + # ref should be a content-ID string like "$1" + assert isinstance(ref, str) + assert ref.startswith("$") + + def test_update_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + cs.records.update("account", "guid-1", {"name": "X"}) + internal = batch._items[0] + assert len(internal.operations) == 1 + + def test_delete_adds_to_changeset(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + cs.records.delete("account", "guid-1") + internal = batch._items[0] + assert len(internal.operations) == 1 + + def test_content_id_increments(self, async_client): + batch = _make_batch(async_client) + cs = batch.changeset() + ref1 = cs.records.create("account", {"name": "A"}) + ref2 = cs.records.create("contact", {"firstname": "B"}) + assert ref1 != ref2 + + +# --------------------------------------------------------------------------- +# AsyncBatchRequest.execute +# --------------------------------------------------------------------------- + + +class TestAsyncBatchRequestExecute: + async def test_execute_calls_batch_client(self, async_client, mock_od): + """execute() delegates to _AsyncBatchClient and returns BatchResult.""" + from PowerPlatform.Dataverse.models.batch import BatchResult, BatchItemResponse + + mock_result = BatchResult(responses=[BatchItemResponse(status_code=204)]) + + # Patch _AsyncBatchClient so we don't need a real HTTP client + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + batch.records.create("account", {"name": "X"}) + result = await batch.execute() + + mock_instance.execute.assert_called_once() + assert isinstance(result, BatchResult) + + async def test_execute_passes_continue_on_error(self, async_client, mock_od): + """execute() passes continue_on_error to _AsyncBatchClient.execute.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + mock_result = BatchResult() + + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + await batch.execute(continue_on_error=True) + + _, kwargs = mock_instance.execute.call_args + assert kwargs["continue_on_error"] is True + + async def test_execute_empty_batch_ok(self, async_client, mock_od): + """execute() with an empty batch does not raise.""" + from PowerPlatform.Dataverse.models.batch import BatchResult + + mock_result = BatchResult() + + with __import__("unittest.mock", fromlist=["patch"]).patch( + "PowerPlatform.Dataverse.aio.operations.async_batch._AsyncBatchClient" + ) as mock_cls: + mock_instance = AsyncMock() + mock_instance.execute.return_value = mock_result + mock_cls.return_value = mock_instance + + batch = _make_batch(async_client) + result = await batch.execute() + + assert isinstance(result, BatchResult) + + +# --------------------------------------------------------------------------- +# Multiple operations in one batch +# --------------------------------------------------------------------------- + + +class TestAsyncBatchMultipleOperations: + def test_multiple_items_accumulated(self, async_client): + batch = _make_batch(async_client) + batch.records.create("account", {"name": "A"}) + with pytest.warns(DeprecationWarning): + batch.records.get("account", "guid-1") + batch.tables.get("account") + batch.query.sql("SELECT name FROM account") + assert len(batch._items) == 4 diff --git a/tests/unit/aio/test_async_client.py b/tests/unit/aio/test_async_client.py new file mode 100644 index 00000000..5997a30a --- /dev/null +++ b/tests/unit/aio/test_async_client.py @@ -0,0 +1,241 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations +from PowerPlatform.Dataverse.aio.operations.async_dataframe import AsyncDataFrameOperations +from PowerPlatform.Dataverse.aio.operations.async_batch import AsyncBatchOperations +from PowerPlatform.Dataverse.core.config import DataverseConfig, OperationContext + + +def _make_credential() -> MagicMock: + return MagicMock(spec=AsyncTokenCredential) + + +class TestAsyncDataverseClientInit: + """Tests for AsyncDataverseClient initialization and validation.""" + + def test_valid_init(self): + """AsyncDataverseClient initializes with valid url and credential.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._base_url == "https://org.crm.dynamics.com" + assert not client._closed + + def test_trailing_slash_stripped(self): + """Trailing slash is stripped from base_url.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com/", _make_credential()) + assert client._base_url == "https://org.crm.dynamics.com" + + def test_empty_base_url_raises(self): + """Empty base_url raises ValueError.""" + with pytest.raises(ValueError, match="base_url is required"): + AsyncDataverseClient("", _make_credential()) + + def test_namespace_attributes_created(self): + """All operation namespace attributes are created.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert isinstance(client.records, AsyncRecordOperations) + assert isinstance(client.tables, AsyncTableOperations) + assert isinstance(client.query, AsyncQueryOperations) + assert isinstance(client.files, AsyncFileOperations) + assert isinstance(client.dataframe, AsyncDataFrameOperations) + assert isinstance(client.batch, AsyncBatchOperations) + + def test_odata_and_session_initially_none(self): + """_odata and _session are None until first use.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._odata is None + assert client._session is None + + +class TestAsyncDataverseClientContextManager: + """Tests for async context manager protocol.""" + + async def test_aenter_returns_self(self): + """__aenter__ returns the client instance.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session_cls.return_value = MagicMock() + result = await client.__aenter__() + assert result is client + + async def test_aenter_creates_session(self): + """__aenter__ creates an aiohttp.ClientSession.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session_cls.return_value = MagicMock() + await client.__aenter__() + mock_session_cls.assert_called_once() + assert client._session is not None + + async def test_aenter_does_not_recreate_existing_session(self): + """__aenter__ does not replace an existing session.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + existing_session = MagicMock() + client._session = existing_session + with patch("aiohttp.ClientSession") as mock_session_cls: + await client.__aenter__() + mock_session_cls.assert_not_called() + assert client._session is existing_session + + async def test_aexit_calls_aclose(self): + """__aexit__ calls aclose() to release resources.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client.aclose = AsyncMock() + await client.__aexit__(None, None, None) + client.aclose.assert_called_once() + + async def test_aenter_raises_after_close(self): + """__aenter__ raises RuntimeError after the client has been closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + await client.__aenter__() + + +class TestAsyncDataverseClientAclose: + """Tests for aclose() lifecycle.""" + + async def test_aclose_sets_closed_flag(self): + """aclose() marks the client as closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + await client.aclose() + assert client._closed + + async def test_aclose_closes_session(self): + """aclose() closes the aiohttp.ClientSession.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + mock_session = MagicMock() + mock_session.close = AsyncMock() + client._session = mock_session + await client.aclose() + mock_session.close.assert_called_once() + assert client._session is None + + async def test_aclose_closes_odata(self): + """aclose() closes the internal _odata client.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + mock_odata = AsyncMock() + client._odata = mock_odata + await client.aclose() + mock_odata.close.assert_called_once() + assert client._odata is None + + async def test_aclose_idempotent(self): + """aclose() is safe to call multiple times.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + await client.aclose() + await client.aclose() # should not raise + assert client._closed + + async def test_context_manager_closes_on_exit(self): + """Using async with calls aclose() on exit.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + with patch("aiohttp.ClientSession") as mock_session_cls: + mock_session = MagicMock() + mock_session.close = AsyncMock() + mock_session_cls.return_value = mock_session + async with client: + pass + assert client._closed + + +class TestAsyncDataverseClientCheckClosed: + """Tests for _check_closed guard.""" + + def test_check_closed_raises_when_closed(self): + """_check_closed() raises RuntimeError when the client is closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + client._check_closed() + + def test_check_closed_does_not_raise_when_open(self): + """_check_closed() does not raise when the client is open.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._check_closed() # should not raise + + +class TestAsyncDataverseClientGetOdata: + """Tests for _get_odata() lazy initialisation of the internal OData client.""" + + async def test_get_odata_creates_client_on_first_call(self): + """_get_odata() instantiates _AsyncODataClient and stores it in _odata on first call.""" + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._odata is None + od = client._get_odata() + assert isinstance(od, _AsyncODataClient) + assert client._odata is od + await client.aclose() + + async def test_get_odata_returns_same_instance(self): + """Subsequent calls to _get_odata() return the same cached instance.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + od1 = client._get_odata() + od2 = client._get_odata() + assert od1 is od2 + await client.aclose() + + +class TestAsyncDataverseClientScopedOdata: + """Tests for _scoped_odata(), an async context manager that guards OData client access.""" + + async def test_scoped_odata_yields_odata_client(self): + """_scoped_odata() yields the low-level _AsyncODataClient instance.""" + from PowerPlatform.Dataverse.aio.data._async_odata import _AsyncODataClient + + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + async with client._scoped_odata() as od: + assert isinstance(od, _AsyncODataClient) + + async def test_scoped_odata_raises_when_closed(self): + """RuntimeError is raised when _scoped_odata() is entered after the client is closed.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + client._closed = True + with pytest.raises(RuntimeError, match="closed"): + async with client._scoped_odata(): + pass + + +class TestAsyncDataverseClientOperationContext: + """Tests for the context= kwarg on AsyncDataverseClient.""" + + def test_context_kwarg_sets_config(self): + """context= stores OperationContext in _config.operation_context.""" + ctx = OperationContext(user_agent_context="app=test/1.0;agent=claude-code") + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential(), context=ctx) + assert client._config.operation_context.user_agent_context == "app=test/1.0;agent=claude-code" + + def test_no_context_leaves_config_default(self): + """Without context=, operation_context defaults to None.""" + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential()) + assert client._config.operation_context is None + + def test_config_and_context_raises(self): + """Providing both config= and context= raises ValueError.""" + ctx = OperationContext(user_agent_context="app=test/1.0") + config = DataverseConfig(operation_context=ctx) + with pytest.raises(ValueError, match="config.*context|context.*config"): + AsyncDataverseClient( + "https://org.crm.dynamics.com", + _make_credential(), + config=config, + context=OperationContext(user_agent_context="app=other/2.0"), + ) + + def test_config_alone_works(self): + """Providing config= without context= uses config's operation_context.""" + ctx = OperationContext(user_agent_context="app=test/1.0;skill=dv") + config = DataverseConfig(operation_context=ctx) + client = AsyncDataverseClient("https://org.crm.dynamics.com", _make_credential(), config=config) + assert client._config.operation_context.user_agent_context == "app=test/1.0;skill=dv" diff --git a/tests/unit/aio/test_async_dataframe.py b/tests/unit/aio/test_async_dataframe.py new file mode 100644 index 00000000..36b69760 --- /dev/null +++ b/tests/unit/aio/test_async_dataframe.py @@ -0,0 +1,183 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +import pandas as pd +from contextlib import asynccontextmanager +from unittest.mock import MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_dataframe import AsyncDataFrameOperations + + +def _make_client_with_od(mock_od): + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped(): + yield mock_od + + client._scoped_odata = _fake_scoped + return client + + +class TestAsyncDataFrameOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.dataframe, AsyncDataFrameOperations) + + +class TestAsyncDataFrameSql: + async def test_sql_returns_dataframe(self, async_client, mock_od): + """sql() executes a SQL query and returns a DataFrame.""" + mock_od._query_sql.return_value = [ + {"name": "Contoso", "accountid": "guid-1"}, + {"name": "Fabrikam", "accountid": "guid-2"}, + ] + + df = await async_client.dataframe.sql("SELECT name FROM account") + + assert isinstance(df, pd.DataFrame) + assert len(df) == 2 + assert "name" in df.columns + + async def test_sql_empty_result_returns_empty_dataframe(self, async_client, mock_od): + """sql() returns an empty DataFrame when no rows match.""" + mock_od._query_sql.return_value = [] + df = await async_client.dataframe.sql("SELECT name FROM account WHERE 1=0") + assert isinstance(df, pd.DataFrame) + assert len(df) == 0 + + +class TestAsyncDataFrameCreate: + async def test_create_returns_series_of_guids(self, async_client, mock_od): + """create() returns a Series of GUIDs aligned with the input DataFrame.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1", "guid-2"] + + df = pd.DataFrame([{"name": "Contoso"}, {"name": "Fabrikam"}]) + result = await async_client.dataframe.create("account", df) + + assert isinstance(result, pd.Series) + assert list(result) == ["guid-1", "guid-2"] + + async def test_create_non_dataframe_raises(self, async_client, mock_od): + """create() raises TypeError if records is not a DataFrame.""" + with pytest.raises(TypeError): + await async_client.dataframe.create("account", [{"name": "X"}]) + + async def test_create_empty_dataframe_raises(self, async_client, mock_od): + """create() raises ValueError if records is empty.""" + with pytest.raises(ValueError): + await async_client.dataframe.create("account", pd.DataFrame()) + + async def test_create_all_null_row_raises(self, async_client, mock_od): + """create() raises ValueError if any row has no non-null values.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + df = pd.DataFrame([{"name": None}]) + with pytest.raises(ValueError, match="no non-null values"): + await async_client.dataframe.create("account", df) + + async def test_create_id_count_mismatch_raises(self, async_client, mock_od): + """create() raises ValueError if the server returns wrong number of IDs.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1"] # 1 ID for 2 rows + + df = pd.DataFrame([{"name": "A"}, {"name": "B"}]) + with pytest.raises(ValueError, match="returned"): + await async_client.dataframe.create("account", df) + + +class TestAsyncDataFrameUpdate: + async def test_update_single_row(self, async_client, mock_od): + """update() with a single-row DataFrame calls records.update once.""" + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": "555"}]) + await async_client.dataframe.update("account", df, id_column="accountid") + mock_od._update.assert_called_once_with("account", "guid-1", {"telephone1": "555"}) + + async def test_update_multiple_rows(self, async_client, mock_od): + """update() with multiple rows calls records.update with lists.""" + df = pd.DataFrame( + [ + {"accountid": "guid-1", "telephone1": "555"}, + {"accountid": "guid-2", "telephone1": "666"}, + ] + ) + await async_client.dataframe.update("account", df, id_column="accountid") + mock_od._update_by_ids.assert_called_once_with( + "account", + ["guid-1", "guid-2"], + [{"telephone1": "555"}, {"telephone1": "666"}], + ) + + async def test_update_non_dataframe_raises(self, async_client, mock_od): + """update() raises TypeError if changes is not a DataFrame.""" + with pytest.raises(TypeError): + await async_client.dataframe.update("account", [{}], id_column="id") + + async def test_update_empty_dataframe_raises(self, async_client, mock_od): + """update() raises ValueError if changes is empty.""" + with pytest.raises(ValueError): + await async_client.dataframe.update("account", pd.DataFrame(), id_column="id") + + async def test_update_missing_id_column_raises(self, async_client, mock_od): + """update() raises ValueError if id_column is not in the DataFrame.""" + df = pd.DataFrame([{"name": "X"}]) + with pytest.raises(ValueError, match="id_column"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_invalid_ids_raises(self, async_client, mock_od): + """update() raises ValueError if id_column contains invalid (non-string) values.""" + df = pd.DataFrame([{"accountid": None, "name": "X"}]) + with pytest.raises(ValueError, match="invalid values"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_no_change_columns_raises(self, async_client, mock_od): + """update() raises ValueError if no columns exist besides id_column.""" + df = pd.DataFrame([{"accountid": "guid-1"}]) + with pytest.raises(ValueError, match="No columns to update"): + await async_client.dataframe.update("account", df, id_column="accountid") + + async def test_update_all_null_rows_skipped(self, async_client, mock_od): + """update() skips rows where all change values are NaN/None.""" + df = pd.DataFrame([{"accountid": "guid-1", "telephone1": None}]) + await async_client.dataframe.update("account", df, id_column="accountid") + # All values are null -> no updates sent + mock_od._update.assert_not_called() + mock_od._update_by_ids.assert_not_called() + + +class TestAsyncDataFrameDelete: + async def test_delete_single(self, async_client, mock_od): + """delete() with a single-element Series calls records.delete once.""" + ids = pd.Series(["guid-1"]) + result = await async_client.dataframe.delete("account", ids) + mock_od._delete.assert_called_once_with("account", "guid-1") + assert result is None + + async def test_delete_multiple_bulk(self, async_client, mock_od): + """delete() with multiple IDs and use_bulk_delete=True uses BulkDelete.""" + mock_od._delete_multiple.return_value = "job-guid" + ids = pd.Series(["guid-1", "guid-2"]) + result = await async_client.dataframe.delete("account", ids) + mock_od._delete_multiple.assert_called_once_with("account", ["guid-1", "guid-2"]) + assert result == "job-guid" + + async def test_delete_empty_series_returns_none(self, async_client, mock_od): + """delete() with an empty Series returns None without calling _delete.""" + result = await async_client.dataframe.delete("account", pd.Series([], dtype=str)) + mock_od._delete.assert_not_called() + assert result is None + + async def test_delete_non_series_raises(self, async_client, mock_od): + """delete() raises TypeError if ids is not a pandas Series.""" + with pytest.raises(TypeError): + await async_client.dataframe.delete("account", ["guid-1", "guid-2"]) + + async def test_delete_invalid_ids_raises(self, async_client, mock_od): + """delete() raises ValueError if any ID is not a non-empty string.""" + ids = pd.Series(["guid-1", None]) + with pytest.raises(ValueError, match="invalid values"): + await async_client.dataframe.delete("account", ids) diff --git a/tests/unit/aio/test_async_files.py b/tests/unit/aio/test_async_files.py new file mode 100644 index 00000000..9c326d07 --- /dev/null +++ b/tests/unit/aio/test_async_files.py @@ -0,0 +1,53 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from PowerPlatform.Dataverse.aio.operations.async_files import AsyncFileOperations + + +class TestAsyncFileOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.files, AsyncFileOperations) + + +class TestAsyncFileUpload: + async def test_upload_delegates_to_upload_file(self, async_client, mock_od): + """upload() calls od._upload_file with all provided arguments.""" + await async_client.files.upload( + "account", + "guid-1", + "new_Document", + "/path/to/file.pdf", + mode="small", + mime_type="application/pdf", + if_none_match=False, + ) + + mock_od._upload_file.assert_called_once_with( + "account", + "guid-1", + "new_Document", + "/path/to/file.pdf", + mode="small", + mime_type="application/pdf", + if_none_match=False, + ) + + async def test_upload_default_args(self, async_client, mock_od): + """upload() passes None/True for optional args when not specified.""" + await async_client.files.upload("account", "guid-1", "new_Doc", "/path/file.txt") + + mock_od._upload_file.assert_called_once_with( + "account", + "guid-1", + "new_Doc", + "/path/file.txt", + mode=None, + mime_type=None, + if_none_match=True, + ) + + async def test_upload_returns_none(self, async_client, mock_od): + """upload() returns None.""" + result = await async_client.files.upload("account", "guid-1", "new_Doc", "/path/file.txt") + assert result is None diff --git a/tests/unit/aio/test_async_query.py b/tests/unit/aio/test_async_query.py new file mode 100644 index 00000000..3d867bd3 --- /dev/null +++ b/tests/unit/aio/test_async_query.py @@ -0,0 +1,542 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import pytest +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock, MagicMock + +from azure.core.credentials_async import AsyncTokenCredential + +from PowerPlatform.Dataverse.aio.async_client import AsyncDataverseClient +from PowerPlatform.Dataverse.aio.operations.async_query import AsyncQueryOperations +from PowerPlatform.Dataverse.aio.models.async_fetchxml_query import AsyncFetchXmlQuery +from PowerPlatform.Dataverse.aio.models.async_query_builder import AsyncQueryBuilder +from PowerPlatform.Dataverse.models.record import QueryResult, Record + + +def _make_async_client_with_od(mock_od): + """Helper: create async client with mocked _scoped_odata.""" + cred = MagicMock(spec=AsyncTokenCredential) + client = AsyncDataverseClient("https://example.crm.dynamics.com", cred) + + @asynccontextmanager + async def _fake_scoped(): + yield mock_od + + client._scoped_odata = _fake_scoped + return client + + +_SIMPLE_FETCHXML = '' + + +class TestAsyncQueryOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.query, AsyncQueryOperations) + + def test_builder_returns_async_query_builder(self, async_client): + """builder() returns an AsyncQueryBuilder bound to this client.""" + qb = async_client.query.builder("account") + assert isinstance(qb, AsyncQueryBuilder) + assert qb._query_ops is async_client.query + + def test_fetchxml_returns_async_fetchxml_query(self, async_client): + """fetchxml() returns an AsyncFetchXmlQuery for valid XML.""" + q = async_client.query.fetchxml(_SIMPLE_FETCHXML) + assert isinstance(q, AsyncFetchXmlQuery) + assert q._entity_name == "account" + + +class TestAsyncQueryBuilder: + async def test_execute_returns_query_result(self, async_client, mock_od): + """builder().execute() collects all pages into a QueryResult.""" + + async def _pages(*args, **kwargs): + yield [{"name": "Contoso", "accountid": "g1"}] + yield [{"name": "Fabrikam", "accountid": "g2"}] + + mock_od._get_multiple = _pages + + result = await async_client.query.builder("account").select("name").execute() + + assert isinstance(result, QueryResult) + assert len(result) == 2 + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_execute_pages_yields_per_page(self, async_client, mock_od): + """builder().execute_pages() yields one QueryResult per page.""" + + async def _pages(*args, **kwargs): + yield [{"name": "A", "accountid": "g1"}] + yield [{"name": "B", "accountid": "g2"}] + + mock_od._get_multiple = _pages + + pages = [] + async for page in async_client.query.builder("account").select("name").execute_pages(): + pages.append(page) + + assert len(pages) == 2 + assert pages[0][0]["name"] == "A" + assert pages[1][0]["name"] == "B" + + async def test_execute_raises_without_scope(self, async_client): + """execute() raises ValueError when no select/where/top/page_size is set.""" + with pytest.raises(ValueError, match="full-table scans"): + await async_client.query.builder("account").execute() + + async def test_execute_raises_when_unbound(self): + """execute() raises RuntimeError when builder was not created via client.query.builder().""" + qb = AsyncQueryBuilder("account") + qb.select("name") + with pytest.raises(RuntimeError, match="client.query.builder"): + await qb.execute() + + async def test_execute_pages_raises_without_scope(self, async_client): + """execute_pages() raises ValueError when no scope constraint is set.""" + with pytest.raises(ValueError, match="full-table scans"): + async for _ in async_client.query.builder("account").execute_pages(): + pass + + def test_chaining_methods_return_self(self, async_client): + """All fluent methods return the same AsyncQueryBuilder instance.""" + from PowerPlatform.Dataverse.models.filters import col + + qb = async_client.query.builder("account") + assert qb.select("name") is qb + assert qb.where(col("statecode") == 0) is qb + assert qb.order_by("name") is qb + assert qb.top(10) is qb + assert qb.page_size(5) is qb + + +class TestAsyncFetchXmlQueryFactory: + def test_fetchxml_invalid_type_raises(self, async_client): + """fetchxml() raises ValidationError when xml is not a string.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError): + async_client.query.fetchxml(123) + + def test_fetchxml_empty_raises(self, async_client): + """fetchxml() raises ValidationError for empty string.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError): + async_client.query.fetchxml(" ") + + def test_fetchxml_malformed_raises(self, async_client): + """fetchxml() raises ValidationError for malformed XML.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + with pytest.raises(ValidationError, match="not well-formed"): + async_client.query.fetchxml("") + + def test_fetchxml_missing_entity_element_raises(self, async_client): + """fetchxml() raises ValueError when element is absent.""" + with pytest.raises(ValueError, match=""): + async_client.query.fetchxml("") + + def test_fetchxml_missing_entity_name_raises(self, async_client): + """fetchxml() raises ValueError when has no name attribute.""" + with pytest.raises(ValueError, match="name"): + async_client.query.fetchxml("") + + +class TestAsyncFetchXmlQueryExecution: + async def test_execute_returns_query_result(self, async_client, mock_od): + """AsyncFetchXmlQuery.execute() collects all pages into a QueryResult.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(return_value=resp) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert isinstance(result, QueryResult) + assert len(result) == 1 + assert result[0]["name"] == "Contoso" + + async def test_execute_pages_yields_pages(self, async_client, mock_od): + """AsyncFetchXmlQuery.execute_pages() yields one QueryResult per page.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(return_value=resp) + + pages = [] + async for page in async_client.query.fetchxml(_SIMPLE_FETCHXML).execute_pages(): + pages.append(page) + + assert len(pages) == 1 + assert pages[0][0]["name"] == "Contoso" + + +class TestAsyncQuerySql: + async def test_sql_returns_records(self, async_client, mock_od): + """sql() calls _query_sql and wraps results in Record objects.""" + mock_od._query_sql.return_value = [ + {"name": "Contoso", "accountid": "guid-1"}, + {"name": "Fabrikam", "accountid": "guid-2"}, + ] + + result = await async_client.query.sql("SELECT TOP 2 name FROM account") + + mock_od._query_sql.assert_called_once_with("SELECT TOP 2 name FROM account") + assert len(result) == 2 + assert all(isinstance(r, Record) for r in result) + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_sql_empty_result(self, async_client, mock_od): + """sql() returns an empty list when no rows match.""" + mock_od._query_sql.return_value = [] + result = await async_client.query.sql("SELECT name FROM account WHERE name = 'X'") + assert result == [] + + +class TestAsyncQuerySqlColumns: + async def test_sql_columns_filters_virtual_and_system(self, async_client, mock_od): + """sql_columns() calls tables.list_columns and filters out virtual/system columns.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "accountid", + "AttributeType": "Uniqueidentifier", + "IsPrimaryId": True, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "versionnumber", + "AttributeType": "BigInt", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + ] + + cols = await async_client.query.sql_columns("account") + + # versionnumber is a system column — excluded by default + names = [c["name"] for c in cols] + assert "versionnumber" not in names + assert "accountid" in names + assert "name" in names + + async def test_sql_columns_include_system(self, async_client, mock_od): + """sql_columns(include_system=True) includes system columns.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "versionnumber", + "AttributeType": "BigInt", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + } + ] + + cols = await async_client.query.sql_columns("account", include_system=True) + assert any(c["name"] == "versionnumber" for c in cols) + + async def test_sql_columns_excludes_attribute_of(self, async_client, mock_od): + """sql_columns() excludes columns where AttributeOf is set.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "parentcustomeridname", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": "parentcustomerid", + } + ] + + cols = await async_client.query.sql_columns("contact") + assert cols == [] + + async def test_sql_columns_skips_empty_logical_name(self, async_client, mock_od): + """sql_columns() skips columns where LogicalName is empty.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": False, + "DisplayName": {}, + "AttributeOf": None, + }, + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + "AttributeOf": None, + }, + ] + cols = await async_client.query.sql_columns("account") + names = [c["name"] for c in cols] + assert "" not in names + assert "name" in names + + async def test_sql_columns_extracts_display_label(self, async_client, mock_od): + """sql_columns() extracts label from UserLocalizedLabel when present.""" + mock_od._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {"UserLocalizedLabel": {"Label": "Account Name", "LanguageCode": 1033}}, + "AttributeOf": None, + }, + ] + cols = await async_client.query.sql_columns("account") + assert len(cols) == 1 + assert cols[0]["label"] == "Account Name" + + +class TestAsyncQueryOdataExpands: + async def test_odata_expands_returns_nav_properties(self, async_client, mock_od): + """odata_expands() returns navigation property metadata.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.return_value = "accounts" + + result = await async_client.query.odata_expands("contact") + + assert len(result) == 1 + assert result[0]["nav_property"] == "parentcustomerid_account" + assert result[0]["target_table"] == "account" + + async def test_odata_expands_filters_non_referencing(self, async_client, mock_od): + """odata_expands() skips relationships where ReferencingEntity != table.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "account", # not "contact" + "ReferencingEntityNavigationPropertyName": "ownerid_systemuser", + "ReferencedEntity": "systemuser", + "ReferencingAttribute": "ownerid", + "SchemaName": "account_owner_rel", + } + ] + mock_od._entity_set_from_schema_name.return_value = "systemusers" + + result = await async_client.query.odata_expands("contact") + assert result == [] + + async def test_odata_expands_skips_empty_nav_prop(self, async_client, mock_od): + """odata_expands() skips relationships with empty nav_prop or target.""" + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "", # empty nav prop + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.return_value = "accounts" + + result = await async_client.query.odata_expands("contact") + assert result == [] + + async def test_odata_expands_handles_entity_set_resolution_failure(self, async_client, mock_od): + """odata_expands() sets target_entity_set to '' when resolution raises.""" + from PowerPlatform.Dataverse.core.errors import MetadataError + + mock_od._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "ReferencedEntity": "account", + "ReferencingAttribute": "parentcustomerid", + "SchemaName": "contact_customer_accounts", + } + ] + mock_od._entity_set_from_schema_name.side_effect = MetadataError("not found") + + result = await async_client.query.odata_expands("contact") + + assert len(result) == 1 + assert result[0]["target_entity_set"] == "" + + +class TestAsyncFetchXmlQueryFactoryUrlLength: + def test_fetchxml_url_too_long_raises(self, async_client): + """fetchxml() raises ValidationError when encoded XML exceeds the URL length limit.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + # Build XML long enough to exceed _MAX_URL_LENGTH when encoded + long_xml = '' + '' * 1200 + "" + with pytest.raises(ValidationError, match="URL length limit"): + async_client.query.fetchxml(long_xml) + + +class TestAsyncFetchXmlQueryPaging: + """Tests for multi-page FetchXML execution paths.""" + + async def test_execute_multi_page_with_cookie(self, async_client, mock_od): + """execute() follows paging cookies across multiple pages.""" + import urllib.parse + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + inner = '' + encoded = urllib.parse.quote(urllib.parse.quote(inner)) + paging_cookie = f'' + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": paging_cookie, + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert result[0]["name"] == "Contoso" + assert result[1]["name"] == "Fabrikam" + + async def test_execute_multi_page_cookie_parse_error_fallback(self, async_client, mock_od): + """execute() falls back to simple paging when the cookie XML is malformed.""" + import warnings + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": "<<>>", + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert any("paging cookie could not be parsed" in str(warning.message) for warning in w) + + async def test_execute_multi_page_no_cookie_simple_paging(self, async_client, mock_od): + """execute() falls back to simple page-number paging when no cookie is returned.""" + import warnings + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + page1 = MagicMock() + page1.json = MagicMock( + return_value={ + "value": [{"name": "Contoso", "accountid": "g1"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + # No fetchxmlpagingcookie key + } + ) + page2 = MagicMock() + page2.json = MagicMock( + return_value={ + "value": [{"name": "Fabrikam", "accountid": "g2"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + ) + mock_od._request = AsyncMock(side_effect=[page1, page2]) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter("always") + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + + assert len(result) == 2 + assert any("simple paging" in str(warning.message) for warning in w) + + async def test_execute_json_parse_error_yields_empty_page(self, async_client, mock_od): + """execute() yields an empty page when the response body cannot be parsed as JSON.""" + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + resp = MagicMock() + resp.json = MagicMock(side_effect=Exception("invalid json")) + mock_od._request = AsyncMock(return_value=resp) + + result = await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() + assert len(result) == 0 + + async def test_execute_raises_on_max_pages_exceeded(self, async_client, mock_od): + """execute() raises ValidationError when paging exceeds the maximum page limit.""" + import urllib.parse + import warnings + from PowerPlatform.Dataverse.core.errors import ValidationError + + mock_od._entity_set_from_schema_name = AsyncMock(return_value="accounts") + + def _make_page_resp(page_num: int): + inner = f'' + encoded = urllib.parse.quote(urllib.parse.quote(inner)) + cookie = f'' + resp = MagicMock() + resp.json = MagicMock( + return_value={ + "value": [{"name": f"Record{page_num}", "accountid": f"g{page_num}"}], + "@Microsoft.Dynamics.CRM.morerecords": True, + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": cookie, + } + ) + return resp + + # Always return morerecords=True to trigger the limit + mock_od._request = AsyncMock(side_effect=lambda *a, **kw: _make_page_resp(1)) + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + with pytest.raises(ValidationError, match="exceeded"): + await async_client.query.fetchxml(_SIMPLE_FETCHXML).execute() diff --git a/tests/unit/aio/test_async_records.py b/tests/unit/aio/test_async_records.py new file mode 100644 index 00000000..13b72582 --- /dev/null +++ b/tests/unit/aio/test_async_records.py @@ -0,0 +1,541 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +import warnings +import pytest +from unittest.mock import MagicMock + +from PowerPlatform.Dataverse.aio.operations.async_records import AsyncRecordOperations +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.upsert import UpsertItem + +# --------------------------------------------------------------------------- +# Async generator helpers used by list/list_pages tests +# --------------------------------------------------------------------------- + + +async def _agen(*pages): + """Yield each argument as one page from an async generator.""" + for p in pages: + yield p + + +class TestAsyncRecordOperationsNamespace: + """Verify the namespace attribute type.""" + + def test_namespace_type(self, async_client): + assert isinstance(async_client.records, AsyncRecordOperations) + + +class TestAsyncRecordCreate: + """Tests for AsyncRecordOperations.create.""" + + async def test_create_single(self, async_client, mock_od): + """create() with a single dict calls _entity_set_from_schema_name and _create.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create.return_value = "guid-123" + + result = await async_client.records.create("account", {"name": "Contoso"}) + + mock_od._entity_set_from_schema_name.assert_called_once_with("account") + mock_od._create.assert_called_once_with("accounts", "account", {"name": "Contoso"}) + assert result == "guid-123" + assert isinstance(result, str) + + async def test_create_bulk(self, async_client, mock_od): + """create() with a list of dicts calls _create_multiple.""" + payloads = [{"name": "A"}, {"name": "B"}] + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = ["guid-1", "guid-2"] + + result = await async_client.records.create("account", payloads) + + mock_od._create_multiple.assert_called_once_with("accounts", "account", payloads) + assert result == ["guid-1", "guid-2"] + + async def test_create_single_non_string_return_raises(self, async_client, mock_od): + """create() raises TypeError if _create returns a non-string.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create.return_value = 12345 + + with pytest.raises(TypeError): + await async_client.records.create("account", {"name": "X"}) + + async def test_create_bulk_non_list_return_raises(self, async_client, mock_od): + """create() raises TypeError if _create_multiple returns a non-list.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + mock_od._create_multiple.return_value = "not-a-list" + + with pytest.raises(TypeError): + await async_client.records.create("account", [{"name": "X"}]) + + async def test_create_invalid_data_type_raises(self, async_client, mock_od): + """create() raises TypeError if data is neither dict nor list.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + with pytest.raises(TypeError): + await async_client.records.create("account", "invalid") + + +class TestAsyncRecordUpdate: + """Tests for AsyncRecordOperations.update.""" + + async def test_update_single(self, async_client, mock_od): + """update() with a str id and dict changes calls _update.""" + await async_client.records.update("account", "guid-1", {"telephone1": "555"}) + mock_od._update.assert_called_once_with("account", "guid-1", {"telephone1": "555"}) + + async def test_update_broadcast(self, async_client, mock_od): + """update() with list of ids and single dict calls _update_by_ids.""" + await async_client.records.update("account", ["id-1", "id-2"], {"statecode": 1}) + mock_od._update_by_ids.assert_called_once_with("account", ["id-1", "id-2"], {"statecode": 1}) + + async def test_update_paired(self, async_client, mock_od): + """update() with list of ids and list of dicts calls _update_by_ids.""" + await async_client.records.update("account", ["id-1", "id-2"], [{"name": "A"}, {"name": "B"}]) + mock_od._update_by_ids.assert_called_once_with("account", ["id-1", "id-2"], [{"name": "A"}, {"name": "B"}]) + + async def test_update_single_non_dict_changes_raises(self, async_client, mock_od): + """update() raises TypeError if ids is str but changes is not a dict.""" + with pytest.raises(TypeError): + await async_client.records.update("account", "guid-1", ["not", "a", "dict"]) + + async def test_update_invalid_ids_type_raises(self, async_client, mock_od): + """update() raises TypeError if ids is neither str nor list.""" + with pytest.raises(TypeError): + await async_client.records.update("account", 12345, {"name": "X"}) + + async def test_update_returns_none(self, async_client, mock_od): + """update() returns None.""" + result = await async_client.records.update("account", "guid-1", {"name": "X"}) + assert result is None + + +class TestAsyncRecordDelete: + """Tests for AsyncRecordOperations.delete.""" + + async def test_delete_single(self, async_client, mock_od): + """delete() with a str id calls _delete and returns None.""" + result = await async_client.records.delete("account", "guid-to-delete") + mock_od._delete.assert_called_once_with("account", "guid-to-delete") + assert result is None + + async def test_delete_bulk(self, async_client, mock_od): + """delete() with a list of ids uses _delete_multiple by default.""" + mock_od._delete_multiple.return_value = "job-guid-456" + result = await async_client.records.delete("account", ["id-1", "id-2", "id-3"]) + mock_od._delete_multiple.assert_called_once_with("account", ["id-1", "id-2", "id-3"]) + assert result == "job-guid-456" + + async def test_delete_bulk_sequential(self, async_client, mock_od): + """delete() with use_bulk_delete=False calls _delete once per id.""" + result = await async_client.records.delete("account", ["id-1", "id-2"], use_bulk_delete=False) + assert mock_od._delete.call_count == 2 + mock_od._delete.assert_any_call("account", "id-1") + mock_od._delete.assert_any_call("account", "id-2") + mock_od._delete_multiple.assert_not_called() + assert result is None + + async def test_delete_empty_list(self, async_client, mock_od): + """delete() with an empty list returns None without calling _delete.""" + result = await async_client.records.delete("account", []) + mock_od._delete.assert_not_called() + mock_od._delete_multiple.assert_not_called() + assert result is None + + async def test_delete_invalid_ids_type_raises(self, async_client, mock_od): + """delete() raises TypeError if ids is neither str nor list.""" + with pytest.raises(TypeError): + await async_client.records.delete("account", 12345) + + async def test_delete_list_with_non_string_guid_raises(self, async_client, mock_od): + """delete() raises TypeError if the ids list contains non-string entries.""" + with pytest.raises(TypeError): + await async_client.records.delete("account", ["valid-guid", 42]) + + +class TestAsyncRecordUpsert: + """Tests for AsyncRecordOperations.upsert.""" + + async def test_upsert_single_upsert_item(self, async_client, mock_od): + """upsert() with a single UpsertItem calls _upsert.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) + + result = await async_client.records.upsert("account", [item]) + + mock_od._upsert.assert_called_once_with( + "accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"} + ) + mock_od._upsert_multiple.assert_not_called() + assert result is None + + async def test_upsert_single_dict(self, async_client, mock_od): + """upsert() with a single dict item calls _upsert.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + item = {"alternate_key": {"accountnumber": "ACC-001"}, "record": {"name": "Contoso"}} + + await async_client.records.upsert("account", [item]) + + mock_od._upsert.assert_called_once_with( + "accounts", "account", {"accountnumber": "ACC-001"}, {"name": "Contoso"} + ) + + async def test_upsert_multiple_calls_upsert_multiple(self, async_client, mock_od): + """upsert() with multiple items calls _upsert_multiple.""" + mock_od._entity_set_from_schema_name.return_value = "accounts" + items = [ + UpsertItem(alternate_key={"accountnumber": "A"}, record={"name": "Contoso"}), + UpsertItem(alternate_key={"accountnumber": "B"}, record={"name": "Fabrikam"}), + ] + + await async_client.records.upsert("account", items) + + mock_od._upsert_multiple.assert_called_once_with( + "accounts", + "account", + [{"accountnumber": "A"}, {"accountnumber": "B"}], + [{"name": "Contoso"}, {"name": "Fabrikam"}], + ) + mock_od._upsert.assert_not_called() + + async def test_upsert_empty_list_raises(self, async_client, mock_od): + """upsert() with an empty list raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", []) + + async def test_upsert_non_list_raises(self, async_client, mock_od): + """upsert() with a non-list argument raises TypeError.""" + item = UpsertItem(alternate_key={"accountnumber": "X"}, record={"name": "Y"}) + with pytest.raises(TypeError): + await async_client.records.upsert("account", item) + + async def test_upsert_invalid_item_raises(self, async_client, mock_od): + """upsert() with an item that is neither UpsertItem nor valid dict raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", [42]) + + async def test_upsert_dict_missing_record_key_raises(self, async_client, mock_od): + """upsert() with a dict missing the 'record' key raises TypeError.""" + with pytest.raises(TypeError): + await async_client.records.upsert("account", [{"alternate_key": {"name": "acc1"}}]) + + +# --------------------------------------------------------------------------- +# retrieve() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordRetrieve: + """Tests for AsyncRecordOperations.retrieve().""" + + async def test_retrieve_returns_record(self, async_client, mock_od): + """retrieve() returns a Record instance.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + result = await async_client.records.retrieve("account", "abc") + assert isinstance(result, Record) + assert result["name"] == "Contoso" + + async def test_retrieve_passes_select(self, async_client, mock_od): + """retrieve() passes select= to _get.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + await async_client.records.retrieve("account", "abc", select=["name"]) + mock_od._get.assert_called_once_with("account", "abc", select=["name"], expand=None, include_annotations=None) + + async def test_retrieve_passes_expand(self, async_client, mock_od): + """retrieve() passes expand= to _get.""" + mock_od._get.return_value = { + "accountid": "abc", + "primarycontactid": {"contactid": "cid", "fullname": "John Doe"}, + } + result = await async_client.records.retrieve("account", "abc", expand=["primarycontactid"]) + mock_od._get.assert_called_once_with( + "account", "abc", select=None, expand=["primarycontactid"], include_annotations=None + ) + assert result["primarycontactid"]["fullname"] == "John Doe" + + async def test_retrieve_passes_select_and_expand(self, async_client, mock_od): + """retrieve() passes both select= and expand= to _get.""" + mock_od._get.return_value = {"name": "Contoso", "primarycontactid": {"fullname": "John"}} + await async_client.records.retrieve("account", "abc", select=["name"], expand=["primarycontactid"]) + mock_od._get.assert_called_once_with( + "account", "abc", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + + async def test_retrieve_passes_include_annotations(self, async_client, mock_od): + """retrieve() passes include_annotations= to _get.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get.return_value = { + "accountid": "abc", + "statuscode": 1, + f"statuscode@{annotation}": "Active", + } + result = await async_client.records.retrieve("account", "abc", include_annotations=annotation) + mock_od._get.assert_called_once_with("account", "abc", select=None, expand=None, include_annotations=annotation) + assert result[f"statuscode@{annotation}"] == "Active" + + async def test_retrieve_no_deprecation_warning(self, async_client, mock_od): + """retrieve() does not emit DeprecationWarning.""" + mock_od._get.return_value = {"accountid": "abc", "name": "Contoso"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await async_client.records.retrieve("account", "abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"retrieve() must not emit DeprecationWarning: {dep}" + + async def test_retrieve_returns_none_on_404(self, async_client, mock_od): + """retrieve() returns None when _get raises HttpError with status 404.""" + mock_od._get.side_effect = HttpError("Not Found", 404) + result = await async_client.records.retrieve("account", "nonexistent") + assert result is None + + async def test_retrieve_reraises_non_404(self, async_client, mock_od): + """retrieve() re-raises HttpError for non-404 status codes.""" + mock_od._get.side_effect = HttpError("Server Error", 500) + with pytest.raises(HttpError): + await async_client.records.retrieve("account", "some-id") + + async def test_retrieve_reraises_non_http_errors(self, async_client, mock_od): + """retrieve() re-raises non-HttpError exceptions unchanged.""" + mock_od._get.side_effect = ValueError("Bad input") + with pytest.raises(ValueError): + await async_client.records.retrieve("account", "some-id") + + async def test_retrieve_record_id_set(self, async_client, mock_od): + """retrieve() sets record.id from the record_id argument.""" + mock_od._get.return_value = {"name": "Contoso"} + record = await async_client.records.retrieve("account", "my-id") + assert record.id == "my-id" + + async def test_retrieve_table_set(self, async_client, mock_od): + """retrieve() sets record.table from the table argument.""" + mock_od._get.return_value = {"name": "Contoso"} + record = await async_client.records.retrieve("account", "my-id") + assert record.table == "account" + + +# --------------------------------------------------------------------------- +# list() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordList: + """Tests for AsyncRecordOperations.list().""" + + async def test_list_returns_query_result(self, async_client, mock_od): + """list() returns a QueryResult.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + result = await async_client.records.list("account") + assert isinstance(result, QueryResult) + + async def test_list_collects_all_pages(self, async_client, mock_od): + """list() collects records from all pages into one QueryResult.""" + mock_od._get_multiple = MagicMock( + return_value=_agen( + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ) + ) + result = await async_client.records.list("account") + assert len(result) == 3 + + async def test_list_no_deprecation_warning(self, async_client, mock_od): + """list() does not emit DeprecationWarning.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await async_client.records.list("account", filter="statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"list() must not emit DeprecationWarning: {dep}" + + async def test_list_passes_string_filter(self, async_client, mock_od): + """list() passes a string filter to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", filter="statecode eq 0") + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_passes_filter_expression(self, async_client, mock_od): + """list() converts a FilterExpression to string before passing to _get_multiple.""" + from PowerPlatform.Dataverse.models.filters import col + + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", filter=col("statecode") == 0) + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_passes_select(self, async_client, mock_od): + """list() passes select= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", select=["name", "revenue"]) + assert mock_od._get_multiple.call_args[1]["select"] == ["name", "revenue"] + + async def test_list_passes_top(self, async_client, mock_od): + """list() passes top= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", top=50) + assert mock_od._get_multiple.call_args[1]["top"] == 50 + + async def test_list_none_filter_passes_none(self, async_client, mock_od): + """list() passes filter=None to _get_multiple when no filter specified.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account") + assert mock_od._get_multiple.call_args[1]["filter"] is None + + async def test_list_result_iterable(self, async_client, mock_od): + """list() result is iterable and contains Record instances.""" + mock_od._get_multiple = MagicMock(return_value=_agen([{"name": "X", "accountid": "1"}])) + result = await async_client.records.list("account") + records = list(result) + assert len(records) == 1 + assert records[0]["name"] == "X" + + async def test_list_result_to_dataframe(self, async_client, mock_od): + """list() result can be converted to a DataFrame.""" + import pandas as pd + + mock_od._get_multiple = MagicMock( + return_value=_agen([{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}]) + ) + df = (await async_client.records.list("account", select=["name"])).to_dataframe() + assert isinstance(df, pd.DataFrame) + assert len(df) == 2 + + async def test_list_passes_orderby(self, async_client, mock_od): + """list() passes orderby= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", orderby=["name asc"]) + assert mock_od._get_multiple.call_args[1]["orderby"] == ["name asc"] + + async def test_list_passes_expand(self, async_client, mock_od): + """list() passes expand= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", expand=["primarycontactid"]) + assert mock_od._get_multiple.call_args[1]["expand"] == ["primarycontactid"] + + async def test_list_passes_page_size(self, async_client, mock_od): + """list() passes page_size= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", page_size=200) + assert mock_od._get_multiple.call_args[1]["page_size"] == 200 + + async def test_list_passes_count(self, async_client, mock_od): + """list() passes count=True to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", count=True) + assert mock_od._get_multiple.call_args[1]["count"] is True + + async def test_list_passes_include_annotations(self, async_client, mock_od): + """list() passes include_annotations= to _get_multiple.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get_multiple = MagicMock(return_value=_agen()) + await async_client.records.list("account", include_annotations=annotation) + assert mock_od._get_multiple.call_args[1]["include_annotations"] == annotation + + +# --------------------------------------------------------------------------- +# list_pages() +# --------------------------------------------------------------------------- + + +class TestAsyncRecordListPages: + """Tests for AsyncRecordOperations.list_pages().""" + + async def test_list_pages_is_async_generator(self, async_client, mock_od): + """list_pages() returns an async generator.""" + import inspect + + mock_od._get_multiple = MagicMock(return_value=_agen()) + result = async_client.records.list_pages("account") + assert inspect.isasyncgen(result) + + async def test_list_pages_yields_query_result_per_page(self, async_client, mock_od): + """list_pages() yields one QueryResult per HTTP page.""" + mock_od._get_multiple = MagicMock( + return_value=_agen([{"name": "A", "accountid": "1"}], [{"name": "B", "accountid": "2"}]) + ) + pages = [] + async for page in async_client.records.list_pages("account"): + pages.append(page) + assert len(pages) == 2 + for page in pages: + assert isinstance(page, QueryResult) + + async def test_list_pages_page_contents(self, async_client, mock_od): + """list_pages() preserves per-page record counts.""" + mock_od._get_multiple = MagicMock( + return_value=_agen( + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ) + ) + pages = [] + async for page in async_client.records.list_pages("account"): + pages.append(page) + assert len(pages[0]) == 1 + assert len(pages[1]) == 2 + + async def test_list_pages_no_deprecation_warning(self, async_client, mock_od): + """list_pages() does not emit DeprecationWarning.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + async for _ in async_client.records.list_pages("account", filter="statecode eq 0"): + pass + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + assert dep == [], f"list_pages() must not emit DeprecationWarning: {dep}" + + async def test_list_pages_passes_filter(self, async_client, mock_od): + """list_pages() passes filter= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", filter="statecode eq 0"): + pass + assert mock_od._get_multiple.call_args[1]["filter"] == "statecode eq 0" + + async def test_list_pages_passes_select(self, async_client, mock_od): + """list_pages() passes select= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", select=["name"]): + pass + assert mock_od._get_multiple.call_args[1]["select"] == ["name"] + + async def test_list_pages_passes_top(self, async_client, mock_od): + """list_pages() passes top= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", top=50): + pass + assert mock_od._get_multiple.call_args[1]["top"] == 50 + + async def test_list_pages_passes_orderby(self, async_client, mock_od): + """list_pages() passes orderby= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", orderby=["name asc"]): + pass + assert mock_od._get_multiple.call_args[1]["orderby"] == ["name asc"] + + async def test_list_pages_passes_expand(self, async_client, mock_od): + """list_pages() passes expand= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", expand=["primarycontactid"]): + pass + assert mock_od._get_multiple.call_args[1]["expand"] == ["primarycontactid"] + + async def test_list_pages_passes_page_size(self, async_client, mock_od): + """list_pages() passes page_size= to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", page_size=200): + pass + assert mock_od._get_multiple.call_args[1]["page_size"] == 200 + + async def test_list_pages_passes_count(self, async_client, mock_od): + """list_pages() passes count=True to _get_multiple.""" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", count=True): + pass + assert mock_od._get_multiple.call_args[1]["count"] is True + + async def test_list_pages_passes_include_annotations(self, async_client, mock_od): + """list_pages() passes include_annotations= to _get_multiple.""" + annotation = "OData.Community.Display.V1.FormattedValue" + mock_od._get_multiple = MagicMock(return_value=_agen()) + async for _ in async_client.records.list_pages("account", include_annotations=annotation): + pass + assert mock_od._get_multiple.call_args[1]["include_annotations"] == annotation diff --git a/tests/unit/aio/test_async_tables.py b/tests/unit/aio/test_async_tables.py new file mode 100644 index 00000000..366f0c74 --- /dev/null +++ b/tests/unit/aio/test_async_tables.py @@ -0,0 +1,314 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + + +from PowerPlatform.Dataverse.aio.operations.async_tables import AsyncTableOperations +from PowerPlatform.Dataverse.models.relationship import RelationshipInfo +from PowerPlatform.Dataverse.models.table_info import AlternateKeyInfo, TableInfo +from PowerPlatform.Dataverse.models.relationship import ( + LookupAttributeMetadata, + OneToManyRelationshipMetadata, + ManyToManyRelationshipMetadata, +) +from PowerPlatform.Dataverse.models.labels import Label, LocalizedLabel + + +def _label(text: str = "Test") -> Label: + return Label(localized_labels=[LocalizedLabel(label=text, language_code=1033)]) + + +def _table_raw(schema_name: str = "new_Product") -> dict: + return { + "table_schema_name": schema_name, + "entity_set_name": "new_products", + "table_logical_name": "new_product", + "metadata_id": "meta-guid-1", + "columns_created": ["new_Price"], + } + + +def _rel_one_to_many_raw() -> dict: + return { + "relationship_id": "rel-guid-1", + "relationship_schema_name": "new_Dept_Emp", + "lookup_schema_name": "new_DeptId", + "referenced_entity": "new_dept", + "referencing_entity": "new_employee", + } + + +def _rel_many_to_many_raw() -> dict: + return { + "relationship_id": "rel-guid-2", + "relationship_schema_name": "new_emp_proj", + "entity1_logical_name": "new_employee", + "entity2_logical_name": "new_project", + } + + +class TestAsyncTableOperationsNamespace: + def test_namespace_type(self, async_client): + assert isinstance(async_client.tables, AsyncTableOperations) + + +class TestAsyncTableCreate: + async def test_create_returns_table_info(self, async_client, mock_od): + """create() returns a TableInfo built from the raw dict.""" + mock_od._create_table.return_value = _table_raw() + columns = {"new_Price": "decimal"} + + result = await async_client.tables.create( + "new_Product", + columns, + solution="MySol", + primary_column="new_ProductName", + display_name="Product", + ) + + mock_od._create_table.assert_called_once_with("new_Product", columns, "MySol", "new_ProductName", "Product") + assert isinstance(result, TableInfo) + assert result.schema_name == "new_Product" + + async def test_create_with_minimal_args(self, async_client, mock_od): + """create() works with only table and columns.""" + mock_od._create_table.return_value = _table_raw() + await async_client.tables.create("new_Product", {}) + mock_od._create_table.assert_called_once_with("new_Product", {}, None, None, None) + + +class TestAsyncTableDelete: + async def test_delete_calls_delete_table(self, async_client, mock_od): + """delete() calls _delete_table with the table schema name.""" + await async_client.tables.delete("new_Product") + mock_od._delete_table.assert_called_once_with("new_Product") + + +class TestAsyncTableGet: + async def test_get_returns_table_info(self, async_client, mock_od): + """get() returns TableInfo when table exists.""" + mock_od._get_table_info.return_value = _table_raw() + result = await async_client.tables.get("new_Product") + assert isinstance(result, TableInfo) + assert result.schema_name == "new_Product" + + async def test_get_returns_none_when_not_found(self, async_client, mock_od): + """get() returns None when _get_table_info returns None.""" + mock_od._get_table_info.return_value = None + result = await async_client.tables.get("new_Product") + assert result is None + + +class TestAsyncTableList: + async def test_list_calls_list_tables(self, async_client, mock_od): + """list() calls _list_tables and returns its result.""" + mock_od._list_tables.return_value = [{"LogicalName": "account"}] + result = await async_client.tables.list() + mock_od._list_tables.assert_called_once_with(filter=None, select=None) + assert result == [{"LogicalName": "account"}] + + async def test_list_with_params(self, async_client, mock_od): + """list() passes filter and select to _list_tables.""" + mock_od._list_tables.return_value = [] + await async_client.tables.list(filter="IsPrivate eq false", select=["LogicalName"]) + mock_od._list_tables.assert_called_once_with(filter="IsPrivate eq false", select=["LogicalName"]) + + +class TestAsyncTableAddColumns: + async def test_add_columns_calls_create_columns(self, async_client, mock_od): + """add_columns() calls _create_columns and returns the result.""" + mock_od._create_columns.return_value = ["new_Notes"] + result = await async_client.tables.add_columns("new_Product", {"new_Notes": "string"}) + mock_od._create_columns.assert_called_once_with("new_Product", {"new_Notes": "string"}) + assert result == ["new_Notes"] + + +class TestAsyncTableRemoveColumns: + async def test_remove_columns_calls_delete_columns(self, async_client, mock_od): + """remove_columns() calls _delete_columns and returns the result.""" + mock_od._delete_columns.return_value = ["new_Notes"] + result = await async_client.tables.remove_columns("new_Product", "new_Notes") + mock_od._delete_columns.assert_called_once_with("new_Product", "new_Notes") + assert result == ["new_Notes"] + + +class TestAsyncTableOneToManyRelationship: + async def test_create_one_to_many(self, async_client, mock_od): + """create_one_to_many_relationship() calls _create_one_to_many_relationship and returns RelationshipInfo.""" + mock_od._create_one_to_many_relationship.return_value = _rel_one_to_many_raw() + + lookup = LookupAttributeMetadata(schema_name="new_DeptId", display_name=_label("Department")) + relationship = OneToManyRelationshipMetadata( + schema_name="new_Dept_Emp", + referenced_entity="new_dept", + referencing_entity="new_employee", + referenced_attribute="new_deptid", + ) + + result = await async_client.tables.create_one_to_many_relationship(lookup, relationship) + + mock_od._create_one_to_many_relationship.assert_called_once_with(lookup, relationship, None) + assert isinstance(result, RelationshipInfo) + assert result.relationship_schema_name == "new_Dept_Emp" + + +class TestAsyncTableManyToManyRelationship: + async def test_create_many_to_many(self, async_client, mock_od): + """create_many_to_many_relationship() calls _create_many_to_many_relationship and returns RelationshipInfo.""" + mock_od._create_many_to_many_relationship.return_value = _rel_many_to_many_raw() + + relationship = ManyToManyRelationshipMetadata( + schema_name="new_emp_proj", + entity1_logical_name="new_employee", + entity2_logical_name="new_project", + ) + + result = await async_client.tables.create_many_to_many_relationship(relationship) + + mock_od._create_many_to_many_relationship.assert_called_once_with(relationship, None) + assert isinstance(result, RelationshipInfo) + assert result.relationship_schema_name == "new_emp_proj" + + +class TestAsyncTableDeleteRelationship: + async def test_delete_relationship(self, async_client, mock_od): + """delete_relationship() calls _delete_relationship with the relationship_id.""" + await async_client.tables.delete_relationship("rel-guid-1") + mock_od._delete_relationship.assert_called_once_with("rel-guid-1") + + +class TestAsyncTableGetRelationship: + async def test_get_relationship_found(self, async_client, mock_od): + """get_relationship() returns RelationshipInfo when found.""" + raw = { + "@odata.type": "#Microsoft.Dynamics.CRM.OneToManyRelationshipMetadata", + "RelationshipId": "rel-guid-1", + "SchemaName": "new_Dept_Emp", + "RelationshipType": "OneToManyRelationship", + "ReferencedEntity": "new_dept", + "ReferencingEntity": "new_employee", + "ReferencingAttribute": "new_deptid", + } + mock_od._get_relationship.return_value = raw + result = await async_client.tables.get_relationship("new_Dept_Emp") + assert isinstance(result, RelationshipInfo) + + async def test_get_relationship_not_found(self, async_client, mock_od): + """get_relationship() returns None when _get_relationship returns None.""" + mock_od._get_relationship.return_value = None + result = await async_client.tables.get_relationship("nonexistent") + assert result is None + + +class TestAsyncTableCreateLookupField: + async def test_create_lookup_field_builds_models_and_delegates(self, async_client, mock_od): + """create_lookup_field() builds lookup/relationship models and calls create_one_to_many_relationship.""" + from unittest.mock import MagicMock + + mock_lookup = LookupAttributeMetadata(schema_name="new_AccountId", display_name=_label("Account")) + mock_rel = OneToManyRelationshipMetadata( + schema_name="new_account_order", + referenced_entity="account", + referencing_entity="new_order", + referenced_attribute="accountid", + ) + # _build_lookup_field_models is a sync method on _ODataBase; use MagicMock so + # od._build_lookup_field_models(...) returns the tuple directly (not a coroutine). + mock_od._build_lookup_field_models = MagicMock(return_value=(mock_lookup, mock_rel)) + mock_od._create_one_to_many_relationship.return_value = { + "relationship_id": "r-guid", + "relationship_schema_name": "new_account_order", + "lookup_schema_name": "new_AccountId", + "referenced_entity": "account", + "referencing_entity": "new_order", + } + + result = await async_client.tables.create_lookup_field( + referencing_table="new_order", + lookup_field_name="new_AccountId", + referenced_table="account", + ) + + mock_od._build_lookup_field_models.assert_called_once() + mock_od._create_one_to_many_relationship.assert_called_once_with(mock_lookup, mock_rel, None) + assert isinstance(result, RelationshipInfo) + + +class TestAsyncTableAlternateKeys: + async def test_create_alternate_key(self, async_client, mock_od): + """create_alternate_key() calls _create_alternate_key and returns AlternateKeyInfo.""" + mock_od._create_alternate_key.return_value = { + "metadata_id": "key-guid", + "schema_name": "new_prod_key", + "key_attributes": ["new_productcode"], + } + + result = await async_client.tables.create_alternate_key( + "new_Product", + "new_prod_key", + ["new_productcode"], + display_name="Product Code", + ) + + mock_od._create_alternate_key.assert_called_once() + assert isinstance(result, AlternateKeyInfo) + assert result.schema_name == "new_prod_key" + assert result.status == "Pending" + + async def test_get_alternate_keys(self, async_client, mock_od): + """get_alternate_keys() calls _get_alternate_keys and returns list of AlternateKeyInfo.""" + mock_od._get_alternate_keys.return_value = [ + { + "MetadataId": "key-guid-1", + "SchemaName": "new_prod_key", + "KeyAttributes": ["new_productcode"], + "EntityKeyIndexStatus": "Active", + } + ] + + result = await async_client.tables.get_alternate_keys("new_Product") + + mock_od._get_alternate_keys.assert_called_once_with("new_Product") + assert len(result) == 1 + assert isinstance(result[0], AlternateKeyInfo) + + async def test_delete_alternate_key(self, async_client, mock_od): + """delete_alternate_key() calls _delete_alternate_key with table and key_id.""" + await async_client.tables.delete_alternate_key("new_Product", "key-guid") + mock_od._delete_alternate_key.assert_called_once_with("new_Product", "key-guid") + + +class TestAsyncTableListColumns: + async def test_list_columns(self, async_client, mock_od): + """list_columns() calls _list_columns and returns its result.""" + mock_od._list_columns.return_value = [{"LogicalName": "name"}] + result = await async_client.tables.list_columns("account") + mock_od._list_columns.assert_called_once_with("account", select=None, filter=None) + assert result == [{"LogicalName": "name"}] + + async def test_list_columns_with_params(self, async_client, mock_od): + """list_columns() passes select and filter to _list_columns.""" + mock_od._list_columns.return_value = [] + await async_client.tables.list_columns( + "account", + select=["LogicalName"], + filter="AttributeType eq 'String'", + ) + mock_od._list_columns.assert_called_once_with( + "account", select=["LogicalName"], filter="AttributeType eq 'String'" + ) + + +class TestAsyncTableListRelationships: + async def test_list_relationships(self, async_client, mock_od): + """list_relationships() calls _list_relationships and returns its result.""" + mock_od._list_relationships.return_value = [{"SchemaName": "new_Dept_Emp"}] + result = await async_client.tables.list_relationships() + mock_od._list_relationships.assert_called_once_with(filter=None, select=None) + assert result == [{"SchemaName": "new_Dept_Emp"}] + + async def test_list_table_relationships(self, async_client, mock_od): + """list_table_relationships() calls _list_table_relationships and returns its result.""" + mock_od._list_table_relationships.return_value = [{"SchemaName": "new_Dept_Emp"}] + result = await async_client.tables.list_table_relationships("account") + mock_od._list_table_relationships.assert_called_once_with("account", filter=None, select=None) + assert result == [{"SchemaName": "new_Dept_Emp"}] diff --git a/tests/unit/core/test_http_errors.py b/tests/unit/core/test_http_errors.py index 39373e05..a0e3f907 100644 --- a/tests/unit/core/test_http_errors.py +++ b/tests/unit/core/test_http_errors.py @@ -3,8 +3,6 @@ import pytest from azure.core.credentials import TokenCredential -from PowerPlatform.Dataverse.client import DataverseClient -from PowerPlatform.Dataverse.core.config import DataverseConfig from PowerPlatform.Dataverse.core.errors import HttpError from PowerPlatform.Dataverse.core._error_codes import HTTP_404, HTTP_429, HTTP_500 from PowerPlatform.Dataverse.data._odata import _ODataClient diff --git a/tests/unit/data/test_batch_edge_cases.py b/tests/unit/data/test_batch_edge_cases.py index 01b4d629..abc3d6c5 100644 --- a/tests/unit/data/test_batch_edge_cases.py +++ b/tests/unit/data/test_batch_edge_cases.py @@ -16,14 +16,12 @@ _BatchClient, _ChangeSet, _ChangeSetBatchItem, - _RecordDelete, _RecordGet, - _extract_boundary, +) +from PowerPlatform.Dataverse.data._batch_base import ( _raise_top_level_batch_error, _split_multipart, _parse_http_response_part, - _CRLF, - _MAX_BATCH_SIZE, ) from PowerPlatform.Dataverse.core.errors import HttpError, ValidationError from PowerPlatform.Dataverse.data._raw_request import _RawRequest diff --git a/tests/unit/data/test_batch_serialization.py b/tests/unit/data/test_batch_serialization.py index 9cdfcb24..dd406d9a 100644 --- a/tests/unit/data/test_batch_serialization.py +++ b/tests/unit/data/test_batch_serialization.py @@ -14,6 +14,7 @@ _RecordCreate, _RecordDelete, _RecordGet, + _RecordList, _RecordUpdate, _RecordUpsert, _TableCreate, @@ -28,6 +29,8 @@ _TableGetRelationship, _TableCreateLookupField, _QuerySql, +) +from PowerPlatform.Dataverse.data._batch_base import ( _extract_boundary, _raise_top_level_batch_error, _parse_mime_part, @@ -252,7 +255,89 @@ def test_resolve_record_get(self): op = _RecordGet(table="account", record_id="guid-1", select=["name"]) result = client._resolve_record_get(op) - od._build_get.assert_called_once_with("account", "guid-1", select=["name"]) + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=None + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_expand(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + op = _RecordGet(table="account", record_id="guid-1", select=["name"], expand=["primarycontactid"]) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_get_with_annotations(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_get.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordGet(table="account", record_id="guid-1", select=["name"], include_annotations=annotation) + result = client._resolve_record_get(op) + + od._build_get.assert_called_once_with( + "account", "guid-1", select=["name"], expand=None, include_annotations=annotation + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_basic(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + op = _RecordList(table="account", select=["name"], filter="statecode eq 0", top=10) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=None, + top=10, + expand=None, + page_size=None, + count=False, + include_annotations=None, + ) + self.assertEqual(result, [mock_req]) + + def test_resolve_record_list_all_params(self): + client, od = self._client_and_od() + mock_req = MagicMock() + od._build_list.return_value = mock_req + + annotation = "OData.Community.Display.V1.FormattedValue" + op = _RecordList( + table="account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) + result = client._resolve_record_list(op) + + od._build_list.assert_called_once_with( + "account", + select=["name"], + filter="statecode eq 0", + orderby=["name asc"], + top=50, + expand=["primarycontactid"], + page_size=100, + count=True, + include_annotations=annotation, + ) self.assertEqual(result, [mock_req]) def test_resolve_record_delete_single(self): @@ -545,7 +630,6 @@ def _make_batch(self): return BatchRecordOperations(batch), batch def test_upsert_single_upsert_item_appended(self): - from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations rec_ops, batch = self._make_batch() item = UpsertItem(alternate_key={"accountnumber": "ACC-001"}, record={"name": "Contoso"}) @@ -559,7 +643,6 @@ def test_upsert_single_upsert_item_appended(self): self.assertEqual(intent.items[0].alternate_key, {"accountnumber": "ACC-001"}) def test_upsert_plain_dict_normalised_to_upsert_item(self): - from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations rec_ops, batch = self._make_batch() rec_ops.upsert("account", [{"alternate_key": {"accountnumber": "X"}, "record": {"name": "Y"}}]) @@ -569,21 +652,18 @@ def test_upsert_plain_dict_normalised_to_upsert_item(self): self.assertEqual(intent.items[0].record, {"name": "Y"}) def test_upsert_empty_list_raises(self): - from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations rec_ops, _ = self._make_batch() with self.assertRaises(TypeError): rec_ops.upsert("account", []) def test_upsert_invalid_item_raises(self): - from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations rec_ops, _ = self._make_batch() with self.assertRaises(TypeError): rec_ops.upsert("account", ["not_a_valid_item"]) def test_upsert_multiple_items_all_normalised(self): - from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations rec_ops, batch = self._make_batch() rec_ops.upsert( @@ -599,6 +679,75 @@ def test_upsert_multiple_items_all_normalised(self): self.assertEqual(intent.items[1].alternate_key, {"accountnumber": "B"}) +class TestBatchRecordOperationsList(unittest.TestCase): + """Tests for BatchRecordOperations.list() surface (operations/batch.py).""" + + def _make_batch(self): + from PowerPlatform.Dataverse.operations.batch import BatchRecordOperations + + batch = MagicMock() + batch._items = [] + return BatchRecordOperations(batch), batch + + def test_list_basic_appends_record_list(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter="statecode eq 0", select=["name"], top=10) + + self.assertEqual(len(batch._items), 1) + intent = batch._items[0] + self.assertIsInstance(intent, _RecordList) + self.assertEqual(intent.table, "account") + self.assertEqual(intent.filter, "statecode eq 0") + self.assertEqual(intent.select, ["name"]) + self.assertEqual(intent.top, 10) + + def test_list_passes_orderby(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", orderby=["name asc"]) + self.assertEqual(batch._items[0].orderby, ["name asc"]) + + def test_list_passes_expand(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", expand=["primarycontactid"]) + self.assertEqual(batch._items[0].expand, ["primarycontactid"]) + + def test_list_passes_page_size(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", page_size=200) + self.assertEqual(batch._items[0].page_size, 200) + + def test_list_passes_count(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account", count=True) + self.assertTrue(batch._items[0].count) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + rec_ops, batch = self._make_batch() + rec_ops.list("account", include_annotations=annotation) + self.assertEqual(batch._items[0].include_annotations, annotation) + + def test_list_filter_expression_converted_to_str(self): + from PowerPlatform.Dataverse.models.filters import col + + rec_ops, batch = self._make_batch() + rec_ops.list("account", filter=col("statecode") == 0) + self.assertEqual(batch._items[0].filter, "statecode eq 0") + + def test_list_defaults(self): + rec_ops, batch = self._make_batch() + rec_ops.list("account") + intent = batch._items[0] + self.assertIsNone(intent.filter) + self.assertIsNone(intent.select) + self.assertIsNone(intent.orderby) + self.assertIsNone(intent.top) + self.assertIsNone(intent.expand) + self.assertIsNone(intent.page_size) + self.assertFalse(intent.count) + self.assertIsNone(intent.include_annotations) + + class TestRaiseTopLevelBatchError(unittest.TestCase): """_raise_top_level_batch_error surfaces Dataverse error details as HttpError.""" diff --git a/tests/unit/data/test_odata_internal.py b/tests/unit/data/test_odata_internal.py index f392ce20..3662a422 100644 --- a/tests/unit/data/test_odata_internal.py +++ b/tests/unit/data/test_odata_internal.py @@ -2,7 +2,6 @@ # Licensed under the MIT license. import json -import time import unittest from enum import Enum from unittest.mock import MagicMock, patch @@ -2051,7 +2050,6 @@ def _bulk_response(*picklists): def test_bulk_fetch_populates_nested_cache(self): """Bulk fetch stores picklists in nested {table: {ts, picklists: {...}}} format.""" - import time resp = self._bulk_response( ("industrycode", [(6, "Technology"), (12, "Consulting")]), diff --git a/tests/unit/data/test_sql_parse.py b/tests/unit/data/test_sql_parse.py index d01ecd00..e95888df 100644 --- a/tests/unit/data/test_sql_parse.py +++ b/tests/unit/data/test_sql_parse.py @@ -513,7 +513,7 @@ def test_extract_pagingcookie_malformed_url_returns_none(): def test_extract_pagingcookie_exception_returns_none(): """Returns None when an unexpected exception is raised during URL parsing (except branch).""" - with patch("PowerPlatform.Dataverse.data._odata.urlparse", side_effect=RuntimeError("boom")): + with patch("PowerPlatform.Dataverse.data._odata_base.urlparse", side_effect=RuntimeError("boom")): assert _extract_pagingcookie("https://org.example/?$skiptoken=x") is None diff --git a/tests/unit/models/test_query_builder.py b/tests/unit/models/test_query_builder.py index 9f094912..e47cb9a6 100644 --- a/tests/unit/models/test_query_builder.py +++ b/tests/unit/models/test_query_builder.py @@ -51,349 +51,120 @@ def test_select_returns_self(self): self.assertIs(qb.select("name"), qb) -class TestComparisonFilters(unittest.TestCase): - """Tests for comparison filter methods.""" +class TestRemovedFilterMethods(unittest.TestCase): + """Verify all 16 filter_* builder methods were removed in 1.0 GA.""" - def test_filter_eq_string(self): - qb = QueryBuilder("account").filter_eq("name", "Contoso") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") - - def test_filter_eq_integer(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) - self.assertEqual(qb.build()["filter"], "statecode eq 0") - - def test_filter_eq_boolean_true(self): - qb = QueryBuilder("account").filter_eq("active", True) - self.assertEqual(qb.build()["filter"], "active eq true") - - def test_filter_eq_boolean_false(self): - qb = QueryBuilder("account").filter_eq("active", False) - self.assertEqual(qb.build()["filter"], "active eq false") - - def test_filter_eq_none(self): - qb = QueryBuilder("account").filter_eq("telephone1", None) - self.assertEqual(qb.build()["filter"], "telephone1 eq null") - - def test_filter_eq_float(self): - qb = QueryBuilder("account").filter_eq("revenue", 1000000.5) - self.assertEqual(qb.build()["filter"], "revenue eq 1000000.5") - - def test_filter_eq_datetime(self): - from datetime import datetime, timezone - - dt = datetime(2024, 1, 15, 10, 30, 0, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_eq("createdon", dt) - self.assertEqual(qb.build()["filter"], "createdon eq 2024-01-15T10:30:00Z") - - -class TestFilterIn(unittest.TestCase): - """Tests for the filter_in() method.""" - - def test_filter_in_integers(self): - qb = QueryBuilder("account").filter_in("statecode", [0, 1, 2]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_strings(self): - qb = QueryBuilder("account").filter_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) - - def test_filter_in_single_value(self): - qb = QueryBuilder("account").filter_in("statecode", [0]) - self.assertEqual( - qb.build()["filter"], - "Microsoft.Dynamics.CRM.In(PropertyName='statecode',PropertyValues=[\"0\"])", - ) - - def test_filter_in_column_lowercased(self): - qb = QueryBuilder("account").filter_in("StateCode", [0, 1]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', - ) - - def test_filter_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_in("statecode", []) - - def test_filter_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_in("statecode", [0, 1]), qb) - - def test_filter_in_with_set(self): - qb = QueryBuilder("account").filter_in("statecode", {0, 1}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.In", result) - self.assertIn("statecode", result) - - def test_filter_in_with_tuple(self): - qb = QueryBuilder("account").filter_in("statecode", (0, 1, 2)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', - ) - - def test_filter_in_int_enum(self): - from enum import IntEnum - - class Priority(IntEnum): - LOW = 1 - HIGH = 3 - - qb = QueryBuilder("account").filter_in("priority", [Priority.LOW, Priority.HIGH]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","3"])', - ) - - def test_filter_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_in("priority", [1, 2, 3]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.In(PropertyName=\'priority\',PropertyValues=["1","2","3"])', - ) - - def test_filter_ne(self): - qb = QueryBuilder("account").filter_ne("statecode", 1) - self.assertEqual(qb.build()["filter"], "statecode ne 1") - - def test_filter_gt(self): - qb = QueryBuilder("account").filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue gt 1000000") - - def test_filter_ge(self): - qb = QueryBuilder("account").filter_ge("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "revenue ge 1000000") - - def test_filter_lt(self): - qb = QueryBuilder("account").filter_lt("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue lt 500000") - - def test_filter_le(self): - qb = QueryBuilder("account").filter_le("revenue", 500000) - self.assertEqual(qb.build()["filter"], "revenue le 500000") - - def test_column_names_lowercased(self): - qb = QueryBuilder("account").filter_eq("StateCode", 0).order_by("Revenue") - params = qb.build() - self.assertEqual(params["filter"], "statecode eq 0") - self.assertEqual(params["orderby"], ["revenue"]) - - def test_string_with_quotes_escaped(self): - qb = QueryBuilder("account").filter_eq("name", "O'Brien's Corp") - self.assertEqual(qb.build()["filter"], "name eq 'O''Brien''s Corp'") - - def test_multiple_filters_and_joined(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_gt("revenue", 1000000) - self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 1000000") - - -class TestStringFunctionFilters(unittest.TestCase): - """Tests for string function filter methods.""" - - def test_filter_contains(self): - qb = QueryBuilder("account").filter_contains("name", "Corp") - self.assertEqual(qb.build()["filter"], "contains(name, 'Corp')") - - def test_filter_startswith(self): - qb = QueryBuilder("account").filter_startswith("name", "Con") - self.assertEqual(qb.build()["filter"], "startswith(name, 'Con')") - - def test_filter_endswith(self): - qb = QueryBuilder("account").filter_endswith("name", "Ltd") - self.assertEqual(qb.build()["filter"], "endswith(name, 'Ltd')") - - def test_filter_contains_single_quotes(self): - qb = QueryBuilder("account").filter_contains("name", "O'Brien") - self.assertEqual(qb.build()["filter"], "contains(name, 'O''Brien')") - - -class TestNullFilters(unittest.TestCase): - """Tests for null/not-null filter methods.""" - - def test_filter_null(self): - qb = QueryBuilder("account").filter_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 eq null") - - def test_filter_not_null(self): - qb = QueryBuilder("account").filter_not_null("telephone1") - self.assertEqual(qb.build()["filter"], "telephone1 ne null") - - -class TestFilterBetween(unittest.TestCase): - """Tests for the filter_between() method.""" - - def test_filter_between_parenthesized(self): - qb = QueryBuilder("account").filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100000 and revenue le 500000)", - ) + def setUp(self): + self.qb = QueryBuilder("account") - def test_filter_between_column_lowercased(self): - qb = QueryBuilder("account").filter_between("Revenue", 100, 500) - self.assertEqual( - qb.build()["filter"], - "(revenue ge 100 and revenue le 500)", - ) - - def test_filter_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_between("revenue", 100, 500), qb) + def test_filter_eq_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_eq("name", "Contoso") - def test_filter_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and (revenue ge 100000 and revenue le 500000)", - ) + def test_filter_ne_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ne("statecode", 1) - def test_filter_between_datetimes(self): - from datetime import datetime, timezone + def test_filter_gt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_gt("revenue", 0) - start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) - end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - qb = QueryBuilder("account").filter_between("createdon", start, end) - self.assertEqual( - qb.build()["filter"], - "(createdon ge 2024-01-01T00:00:00Z and createdon le 2024-12-31T23:59:59Z)", - ) + def test_filter_ge_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_ge("revenue", 0) + def test_filter_lt_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_lt("revenue", 0) -class TestFilterNotIn(unittest.TestCase): - """Tests for the filter_not_in() method.""" + def test_filter_le_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_le("revenue", 0) - def test_filter_not_in_ints(self): - qb = QueryBuilder("account").filter_not_in("statecode", [2, 3]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) + def test_filter_contains_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_contains("name", "Corp") - def test_filter_not_in_strings(self): - qb = QueryBuilder("account").filter_not_in("name", ["Contoso", "Fabrikam"]) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'name\',PropertyValues=["Contoso","Fabrikam"])', - ) + def test_filter_startswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_startswith("name", "Con") - def test_filter_not_in_empty_raises(self): - with self.assertRaises(ValueError): - QueryBuilder("account").filter_not_in("statecode", []) + def test_filter_endswith_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_endswith("name", "Ltd") - def test_filter_not_in_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_in("statecode", [0, 1]), qb) + def test_filter_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_null("telephone1") - def test_filter_not_in_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_in("priority", [1, 2]) - self.assertEqual( - qb.build()["filter"], - 'statecode eq 0 and Microsoft.Dynamics.CRM.NotIn(PropertyName=\'priority\',PropertyValues=["1","2"])', - ) + def test_filter_not_null_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_null("telephone1") - def test_filter_not_in_with_set(self): - qb = QueryBuilder("account").filter_not_in("statecode", {2, 3}) - result = qb.build()["filter"] - self.assertIn("Microsoft.Dynamics.CRM.NotIn", result) - self.assertIn("statecode", result) + def test_filter_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_in("statecode", [0, 1]) - def test_filter_not_in_with_tuple(self): - qb = QueryBuilder("account").filter_not_in("statecode", (2, 3)) - self.assertEqual( - qb.build()["filter"], - 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', - ) + def test_filter_not_in_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_in("statecode", [0, 1]) + def test_filter_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_between("revenue", 100, 500) -class TestFilterNotBetween(unittest.TestCase): - """Tests for the filter_not_between() method.""" + def test_filter_not_between_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_not_between("revenue", 100, 500) - def test_filter_not_between_ints(self): - qb = QueryBuilder("account").filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "not ((revenue ge 100000 and revenue le 500000))", - ) - - def test_filter_not_between_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_not_between("revenue", 100, 500), qb) - - def test_filter_not_between_combined_with_other_filters(self): - qb = QueryBuilder("account").filter_eq("statecode", 0).filter_not_between("revenue", 100000, 500000) - self.assertEqual( - qb.build()["filter"], - "statecode eq 0 and not ((revenue ge 100000 and revenue le 500000))", - ) - - -class TestFilterRaw(unittest.TestCase): - """Tests for the filter_raw() method.""" - - def test_filter_raw(self): - qb = QueryBuilder("account").filter_raw("(statecode eq 0 or statecode eq 1)") - self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") - - def test_filter_raw_returns_self(self): - qb = QueryBuilder("account") - self.assertIs(qb.filter_raw("a eq 1"), qb) - - def test_build_with_plain_string_filter_part(self): - """build() handles plain string entries in _filter_parts (internal path).""" - qb = QueryBuilder("account") - qb._filter_parts.append("name eq 'Contoso'") - self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + def test_filter_raw_removed(self): + with self.assertRaises(AttributeError): + self.qb.filter_raw("statecode eq 0") class TestWhere(unittest.TestCase): """Tests for the where() method with composable expressions.""" def test_where_simple(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)) + qb = QueryBuilder("account").where(col("statecode") == 0) self.assertEqual(qb.build()["filter"], "statecode eq 0") def test_where_complex(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + expr = ((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) - def test_where_combined_with_filter_methods(self): - from PowerPlatform.Dataverse.models.filters import gt + def test_where_combined_with_raw(self): + from PowerPlatform.Dataverse.models.filters import col, raw - qb = QueryBuilder("account").filter_eq("statecode", 0).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(raw("statecode eq 0")).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_multiple_calls(self): - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("statecode", 0)).where(gt("revenue", 100000)) + qb = QueryBuilder("account").where(col("statecode") == 0).where(col("revenue") > 100000) self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") def test_where_preserves_call_order(self): - """Interleaved filter_*() and where() should preserve call order.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + """Multiple where() calls preserve insertion order.""" + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(eq("a", 1)).filter_eq("b", 2).where(gt("c", 3)) + qb = QueryBuilder("account").where(col("a") == 1).where(col("b") == 2).where(col("c") > 3) self.assertEqual(qb.build()["filter"], "a eq 1 and b eq 2 and c gt 3") def test_where_returns_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") - self.assertIs(qb.where(eq("statecode", 0)), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) def test_where_non_expression_raises(self): qb = QueryBuilder("account") @@ -401,21 +172,54 @@ def test_where_non_expression_raises(self): qb.where("statecode eq 0") # type: ignore def test_where_with_not(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col - qb = QueryBuilder("account").where(~eq("statecode", 1)) + qb = QueryBuilder("account").where(~(col("statecode") == 1)) self.assertEqual(qb.build()["filter"], "not (statecode eq 1)") def test_where_with_filter_in(self): - from PowerPlatform.Dataverse.models.filters import filter_in, gt + from PowerPlatform.Dataverse.models.filters import col - expr = filter_in("statecode", [0, 1]) & gt("revenue", 100000) + expr = col("statecode").in_([0, 1]) & (col("revenue") > 100000) qb = QueryBuilder("account").where(expr) self.assertEqual( qb.build()["filter"], '(Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"]) and revenue gt 100000)', ) + def test_where_with_raw_preserves_string(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("(statecode eq 0 or statecode eq 1)")) + self.assertEqual(qb.build()["filter"], "(statecode eq 0 or statecode eq 1)") + + def test_where_negation_of_in(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("statecode").not_in([2, 3])) + self.assertEqual( + qb.build()["filter"], + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_where_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_where_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account").where(col("revenue").not_between(100000, 500000)) + self.assertEqual( + qb.build()["filter"], + "not ((revenue ge 100000 and revenue le 500000))", + ) + class TestOrderBy(unittest.TestCase): """Tests for the order_by() method.""" @@ -635,11 +439,13 @@ def test_empty_builder_only_has_table(self): self.assertNotIn("page_size", params) def test_full_query_build(self): + from PowerPlatform.Dataverse.models.filters import col, raw + qb = ( QueryBuilder("account") .select("name", "revenue", "telephone1") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(raw("statecode eq 0")) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .order_by("name") .expand("primarycontactid") @@ -663,33 +469,34 @@ def test_build_returns_fresh_lists(self): self.assertEqual(params1["select"], params2["select"]) self.assertIsNot(params1["select"], params2["select"]) + def test_build_with_plain_string_filter_part(self): + """build() handles plain string entries in _filter_parts (internal path).""" + qb = QueryBuilder("account") + qb._filter_parts.append("name eq 'Contoso'") + self.assertEqual(qb.build()["filter"], "name eq 'Contoso'") + + def test_build_mixed_string_and_expression_filter_parts(self): + """build() AND-joins raw strings and FilterExpression objects in order.""" + from PowerPlatform.Dataverse.models.filters import col + + qb = QueryBuilder("account") + qb._filter_parts.append("statecode eq 0") + qb.where(col("revenue") > 100000) + self.assertEqual(qb.build()["filter"], "statecode eq 0 and revenue gt 100000") + class TestMethodChainingReturnsSelf(unittest.TestCase): - """Verify all methods return self for chaining.""" + """Verify all public methods return self for chaining.""" def test_all_methods_return_self(self): - from PowerPlatform.Dataverse.models.filters import eq + from PowerPlatform.Dataverse.models.filters import col qb = QueryBuilder("account") self.assertIs(qb.select("name"), qb) - self.assertIs(qb.filter_eq("a", 1), qb) - self.assertIs(qb.filter_ne("b", 2), qb) - self.assertIs(qb.filter_gt("c", 3), qb) - self.assertIs(qb.filter_ge("d", 4), qb) - self.assertIs(qb.filter_lt("e", 5), qb) - self.assertIs(qb.filter_le("f", 6), qb) - self.assertIs(qb.filter_contains("g", "x"), qb) - self.assertIs(qb.filter_startswith("h", "y"), qb) - self.assertIs(qb.filter_endswith("i", "z"), qb) - self.assertIs(qb.filter_null("j"), qb) - self.assertIs(qb.filter_not_null("k"), qb) - self.assertIs(qb.filter_raw("l eq 1"), qb) - self.assertIs(qb.filter_in("m", [1, 2]), qb) - self.assertIs(qb.filter_between("n", 1, 10), qb) - self.assertIs(qb.where(eq("o", 1)), qb) - self.assertIs(qb.order_by("p"), qb) - self.assertIs(qb.expand("q"), qb) + self.assertIs(qb.where(col("statecode") == 0), qb) + self.assertIs(qb.order_by("name"), qb) + self.assertIs(qb.expand("primarycontactid"), qb) self.assertIs(qb.top(10), qb) self.assertIs(qb.page_size(5), qb) self.assertIs(qb.count(), qb) @@ -701,26 +508,40 @@ class TestExecute(unittest.TestCase): """Tests for the execute() terminal method.""" def test_execute_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) with self.assertRaises(RuntimeError) as ctx: qb.execute() self.assertIn("client.query.builder()", str(ctx.exception)) - def test_execute_calls_records_get(self): - """execute() should delegate to client.records.get() with built params.""" + def _make_od(self, pages=None): + """Return (mock_query_ops, mock_od) with _get_multiple pre-wired.""" mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "Test"}]]) + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + def test_execute_calls_get_multiple(self): + """execute() calls _get_multiple() via _scoped_odata with all built params.""" + from PowerPlatform.Dataverse.models.filters import raw + + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -733,10 +554,7 @@ def test_execute_calls_records_get(self): ) def test_execute_returns_flat_records_by_default(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) - + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") @@ -748,66 +566,47 @@ def test_execute_returns_flat_records_by_default(self): self.assertEqual(records[2]["name"], "C") def test_execute_by_page_returns_pages(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - - page1 = [{"name": "A"}, {"name": "B"}] - page2 = [{"name": "C"}] - mock_client.records.get.return_value = iter([page1, page2]) + from PowerPlatform.Dataverse.models.record import QueryResult + mock_query_ops, _ = self._make_od([[{"name": "A"}, {"name": "B"}], [{"name": "C"}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") pages = list(qb.execute(by_page=True)) self.assertEqual(len(pages), 2) - self.assertEqual(pages[0], page1) - self.assertEqual(pages[1], page2) - - def test_execute_unbounded_raises(self): - """execute() with no select/filter/top should raise ValueError.""" - mock_query_ops = MagicMock() - qb = QueryBuilder("account") - qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: - qb.execute() - self.assertIn("Unbounded query", str(ctx.exception)) + self.assertIsInstance(pages[0], QueryResult) + self.assertEqual(len(pages[0]), 2) + self.assertEqual(len(pages[1]), 1) def test_execute_with_only_select_succeeds(self): """execute() with select only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name") - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_filter_succeeds(self): """execute() with filter only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import raw + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_eq("statecode", 0) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.where(raw("statecode eq 0")) + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_top_succeeds(self): """execute() with top only should not raise.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.top(10) - list(qb.execute()) # should not raise - mock_client.records.get.assert_called_once() + qb.execute() # should not raise + mock_od._get_multiple.assert_called_once() def test_execute_with_only_expand_raises(self): """expand alone is not a sufficient constraint.""" @@ -828,51 +627,42 @@ def test_execute_with_only_count_raises(self): qb.execute() def test_execute_with_where_expressions(self): - from PowerPlatform.Dataverse.models.filters import eq, gt - - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import col + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) - list(qb.execute()) + qb.where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", ) def test_execute_with_filter_in(self): - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) + from PowerPlatform.Dataverse.models.filters import col + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.filter_in("statecode", [0, 1, 2]) - list(qb.execute()) + qb.where(col("statecode").in_([0, 1, 2])) + qb.execute() - call_args = mock_client.records.get.call_args self.assertEqual( - call_args.kwargs["filter"], + mock_od._get_multiple.call_args.kwargs["filter"], 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1","2"])', ) def test_execute_passes_count_and_annotations(self): - """execute() should forward count and include_annotations when set.""" - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.records.get.return_value = iter([]) - + """execute() should forward count and include_annotations to _get_multiple.""" + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() - list(qb.execute()) + qb.execute() - mock_client.records.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -885,64 +675,141 @@ def test_execute_passes_count_and_annotations(self): ) -class TestToDataframe(unittest.TestCase): - """Tests for the to_dataframe() terminal method.""" +class TestExecutePages(unittest.TestCase): + """Tests for execute_pages() — lazy per-page QueryResult iterator.""" - def test_to_dataframe_without_query_ops_raises(self): - qb = QueryBuilder("account").filter_eq("statecode", 0) - with self.assertRaises(RuntimeError) as ctx: - qb.to_dataframe() - self.assertIn("client.query.builder()", str(ctx.exception)) + def _make_qb(self): + mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}], [{"name": "B"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + return qb, mock_query_ops - def test_to_dataframe_delegates_to_dataframe_get(self): - """to_dataframe() should delegate to client.dataframe.get() with built params.""" - import pandas as pd + def test_execute_pages_returns_iterator(self): + qb, _ = self._make_qb() + result = qb.execute_pages() + import types - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - mock_client.dataframe.get.return_value = expected_df + self.assertIsInstance(result, types.GeneratorType) + + def test_execute_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + def test_execute_pages_page_contents(self): + qb, _ = self._make_qb() + pages = list(qb.execute_pages()) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + def test_execute_pages_without_query_ops_raises(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) + with self.assertRaises(RuntimeError): + list(qb.execute_pages()) + + def test_execute_pages_without_constraints_raises(self): + mock_query_ops = MagicMock() qb = QueryBuilder("account") qb._query_ops = mock_query_ops - qb.select("name", "revenue").filter_eq("statecode", 0).order_by("revenue", descending=True).top(100).page_size( - 50 - ).expand("primarycontactid") + with self.assertRaises(ValueError): + list(qb.execute_pages()) - result = qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( - "account", - select=["name", "revenue"], - filter="statecode eq 0", - orderby=["revenue desc"], - top=100, - expand=["primarycontactid"], - page_size=50, - count=False, - include_annotations=None, - ) - pd.testing.assert_frame_equal(result, expected_df) +class TestByPageWarning(unittest.TestCase): + """execute(by_page=...) fires UserWarning; plain execute() does not.""" - def test_to_dataframe_unbounded_raises(self): - """to_dataframe() with no select/filter/top should raise ValueError.""" + def _make_qb(self): mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter([[{"name": "A"}]]) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od qb = QueryBuilder("account") qb._query_ops = mock_query_ops - with self.assertRaises(ValueError) as ctx: + qb.select("name") + return qb + + def test_execute_no_flag_no_warning(self): + """execute() with no by_page argument fires no warning.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb.execute() + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 0) + + def test_execute_by_page_true_fires_user_warning(self): + """execute(by_page=True) fires UserWarning pointing to execute_pages().""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + list(qb.execute(by_page=True)) + self.assertIn("execute_pages()", str(ctx.warning)) + + def test_execute_by_page_false_fires_user_warning(self): + """execute(by_page=False) fires UserWarning — redundant flag.""" + qb = self._make_qb() + with self.assertWarns(UserWarning) as ctx: + qb.execute(by_page=False) + self.assertIn("redundant", str(ctx.warning)) + + def test_execute_by_page_true_still_functional(self): + """execute(by_page=True) still returns the raw page iterator.""" + import warnings + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=True) + pages = list(result) + self.assertEqual(len(pages), 1) + + def test_execute_by_page_false_still_returns_query_result(self): + """execute(by_page=False) still returns QueryResult.""" + import warnings + + from PowerPlatform.Dataverse.models.record import QueryResult + + qb = self._make_qb() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", UserWarning) + result = qb.execute(by_page=False) + self.assertIsInstance(result, QueryResult) + + +class TestToDataframe(unittest.TestCase): + """Tests for the to_dataframe() terminal method.""" + + def _make_od(self, pages=None): + mock_query_ops = MagicMock() + mock_od = MagicMock() + mock_od._get_multiple.side_effect = lambda *a, **kw: iter(pages or []) + mock_query_ops._client._scoped_odata.return_value.__enter__.return_value = mock_od + return mock_query_ops, mock_od + + def test_to_dataframe_without_query_ops_raises(self): + from PowerPlatform.Dataverse.models.filters import raw + + qb = QueryBuilder("account").where(raw("statecode eq 0")) + with self.assertRaises(RuntimeError) as ctx: qb.to_dataframe() - self.assertIn("Unbounded query", str(ctx.exception)) + self.assertIn("client.query.builder()", str(ctx.exception)) def test_to_dataframe_returns_dataframe(self): - """to_dataframe() should return a pandas DataFrame.""" + """to_dataframe() collects execute() results into a DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame( - [{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}] - ) - + mock_query_ops, _ = self._make_od([[{"name": "A", "revenue": 100}, {"name": "B", "revenue": 200}]]) qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name", "revenue") @@ -953,20 +820,63 @@ def test_to_dataframe_returns_dataframe(self): self.assertEqual(len(result), 2) self.assertListEqual(list(result.columns), ["name", "revenue"]) - def test_to_dataframe_forwards_count_and_annotations(self): - """to_dataframe() should forward count and include_annotations when set.""" + def test_to_dataframe_empty_result_returns_empty_dataframe(self): + """to_dataframe() with no matching records returns empty DataFrame.""" import pandas as pd - mock_query_ops = MagicMock() - mock_client = mock_query_ops._client - mock_client.dataframe.get.return_value = pd.DataFrame() + mock_query_ops, _ = self._make_od() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 0) + + def test_to_dataframe_calls_get_multiple_with_params(self): + """to_dataframe() passes all built query params to _get_multiple.""" + import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw + + mock_query_ops, mock_od = self._make_od([[{"name": "Contoso", "revenue": 1000}]]) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + ( + qb.select("name", "revenue") + .where(raw("statecode eq 0")) + .order_by("revenue", descending=True) + .top(100) + .page_size(50) + .expand("primarycontactid") + ) + result = qb.to_dataframe() + + mock_od._get_multiple.assert_called_once_with( + "account", + select=["name", "revenue"], + filter="statecode eq 0", + orderby=["revenue desc"], + top=100, + expand=["primarycontactid"], + page_size=50, + count=False, + include_annotations=None, + ) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") + + def test_to_dataframe_forwards_count_and_annotations(self): + """to_dataframe() should forward count and include_annotations when set.""" + mock_query_ops, mock_od = self._make_od() qb = QueryBuilder("account") qb._query_ops = mock_query_ops qb.select("name").count().include_formatted_values() qb.to_dataframe() - mock_client.dataframe.get.assert_called_once_with( + mock_od._get_multiple.assert_called_once_with( "account", select=["name"], filter=None, @@ -978,6 +888,42 @@ def test_to_dataframe_forwards_count_and_annotations(self): include_annotations="OData.Community.Display.V1.FormattedValue", ) + def test_to_dataframe_with_record_objects(self): + """to_dataframe() handles Record objects (with .data attribute).""" + import pandas as pd + + mock_query_ops, _ = self._make_od( + [ + [ + {"name": "Contoso", "revenue": 1000}, + {"name": "Fabrikam", "revenue": 2000}, + ] + ] + ) + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name", "revenue") + + result = qb.to_dataframe() + + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 2) + self.assertEqual(result.iloc[0]["name"], "Contoso") + self.assertEqual(result.iloc[1]["revenue"], 2000) + + def test_to_dataframe_emits_deprecation_warning(self): + """QueryBuilder.to_dataframe() fires DeprecationWarning; use execute().to_dataframe() instead.""" + mock_query_ops, _ = self._make_od() + qb = QueryBuilder("account") + qb._query_ops = mock_query_ops + qb.select("name") + + with self.assertWarns(DeprecationWarning) as ctx: + qb.to_dataframe() + + self.assertIn("QueryBuilder.to_dataframe()", str(ctx.warning)) + self.assertIn("execute().to_dataframe()", str(ctx.warning)) + if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py index 52d5a716..2ed03ddd 100644 --- a/tests/unit/test_client.py +++ b/tests/unit/test_client.py @@ -9,135 +9,87 @@ from PowerPlatform.Dataverse.client import DataverseClient -class TestDataverseClient(unittest.TestCase): +class TestDataverseClientConstruction(unittest.TestCase): + """Tests for DataverseClient construction and lifecycle.""" + + def test_empty_base_url_raises(self): + """DataverseClient raises ValueError when base_url is empty.""" + mock_credential = MagicMock(spec=TokenCredential) + with self.assertRaises(ValueError): + DataverseClient("", mock_credential) + + def test_trailing_slash_stripped(self): + """DataverseClient strips trailing slash from base_url.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com/", mock_credential) + self.assertEqual(client._base_url, "https://example.crm.dynamics.com") + + def test_namespace_attributes_present(self): + """Client exposes records, query, tables, files, dataframe, batch namespaces.""" + mock_credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", mock_credential) + for attr in ("records", "query", "tables", "files", "dataframe", "batch"): + self.assertTrue(hasattr(client, attr), f"Missing namespace: {attr}") + + +class TestRemovedClientMethods(unittest.TestCase): + """Verify all 12 deprecated flat methods were removed from DataverseClient in 1.0 GA. + + These methods previously delegated to namespace equivalents (records.*, query.*, + tables.*, files.*). They were fully removed; callers must use the namespaces directly. + """ + def setUp(self): - """Set up test fixtures before each test method.""" - # Create mock credential self.mock_credential = MagicMock(spec=TokenCredential) - self.base_url = "https://example.crm.dynamics.com" + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Initialize the client under test - self.client = DataverseClient(self.base_url, self.mock_credential) + def test_create_removed(self): + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "Test"}) - # Mock the internal _odata client - # This ensures we verify logic without making actual HTTP calls - self.client._odata = MagicMock() - - def test_create_single(self): - """Test create method with a single record.""" - # Setup mock return values - # _create must return a GUID string - self.client._odata._create.return_value = "00000000-0000-0000-0000-000000000000" - # _entity_set_from_schema_name should return the plural entity set name - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", {"name": "Contoso Ltd"}) - - # Verify - # Ensure _entity_set_from_schema_name was called and its result ("accounts") was passed to _create - self.client._odata._create.assert_called_once_with("accounts", "account", {"name": "Contoso Ltd"}) - - def test_create_multiple(self): - """Test create method with multiple records.""" - payloads = [{"name": "Company A"}, {"name": "Company B"}, {"name": "Company C"}] - - # Setup mock return values - # _create_multiple must return a list of GUID strings - self.client._odata._create_multiple.return_value = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - "00000000-0000-0000-0000-000000000003", - ] - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - - # Execute test - self.client.create("account", payloads) - - # Verify - self.client._odata._create_multiple.assert_called_once_with("accounts", "account", payloads) - - def test_update_single(self): - """Test update method with a single record.""" - self.client.update("account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"}) - self.client._odata._update.assert_called_once_with( - "account", "00000000-0000-0000-0000-000000000000", {"telephone1": "555-0199"} - ) + def test_update_removed(self): + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"name": "Test"}) - def test_update_multiple(self): - """Test update method with multiple records (broadcast).""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - changes = {"statecode": 1} - - self.client.update("account", ids, changes) - self.client._odata._update_by_ids.assert_called_once_with("account", ids, changes) - - def test_delete_single(self): - """Test delete method with a single record.""" - self.client.delete("account", "00000000-0000-0000-0000-000000000000") - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000") - - def test_delete_multiple(self): - """Test delete method with multiple records.""" - ids = [ - "00000000-0000-0000-0000-000000000001", - "00000000-0000-0000-0000-000000000002", - ] - # Mock return value for bulk delete job ID - self.client._odata._delete_multiple.return_value = "job-guid-123" - - job_id = self.client.delete("account", ids) - - self.client._odata._delete_multiple.assert_called_once_with("account", ids) - self.assertEqual(job_id, "job-guid-123") - - def test_get_single(self): - """Test get method with a single record ID.""" - # Setup mock return value - expected_record = {"accountid": "00000000-0000-0000-0000-000000000000", "name": "Contoso"} - self.client._odata._get.return_value = expected_record - - result = self.client.get("account", "00000000-0000-0000-0000-000000000000") - - self.client._odata._get.assert_called_once_with("account", "00000000-0000-0000-0000-000000000000", select=None) - self.assertEqual(result["accountid"], "00000000-0000-0000-0000-000000000000") - self.assertEqual(result["name"], "Contoso") - - def test_get_multiple(self): - """Test get method for querying multiple records.""" - # Setup mock return value (iterator) - expected_batch = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([expected_batch]) - - # Execute query - result_iterator = self.client.get("account", filter="statecode eq 0", top=10) - - # Consume iterator to verify content - results = list(result_iterator) - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - self.assertEqual(len(results), 1) - self.assertEqual(len(results[0]), 2) - self.assertEqual(results[0][0]["name"], "A") - self.assertEqual(results[0][1]["name"], "B") + def test_delete_removed(self): + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") - def test_empty_base_url_raises(self): - """DataverseClient raises ValueError when base_url is empty.""" - with self.assertRaises(ValueError): - DataverseClient("", self.mock_credential) + def test_get_removed(self): + with self.assertRaises(AttributeError): + self.client.get("account", "guid-1") + + def test_query_sql_removed(self): + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") + + def test_get_table_info_removed(self): + with self.assertRaises(AttributeError): + self.client.get_table_info("account") + + def test_create_table_removed(self): + with self.assertRaises(AttributeError): + self.client.create_table("new_Test", {}) + + def test_delete_table_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_table("new_Test") + + def test_list_tables_removed(self): + with self.assertRaises(AttributeError): + self.client.list_tables() + + def test_create_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.create_columns("account", {}) + + def test_delete_columns_removed(self): + with self.assertRaises(AttributeError): + self.client.delete_columns("account", []) + + def test_upload_file_removed(self): + with self.assertRaises(AttributeError): + self.client.upload_file("account", "guid-1", "file_col", "/path/file.pdf") class TestCreateLookupField(unittest.TestCase): diff --git a/tests/unit/test_client_deprecations.py b/tests/unit/test_client_deprecations.py index fbf1382d..b11486be 100644 --- a/tests/unit/test_client_deprecations.py +++ b/tests/unit/test_client_deprecations.py @@ -1,12 +1,11 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT license. -"""Tests for deprecated flat methods on DataverseClient. +"""Tests confirming all 12 deprecated flat methods were removed from DataverseClient. -Each deprecated method on the client should: -1. Emit a DeprecationWarning. -2. Delegate to the correct namespace method (records / query / tables / files). -3. Return the expected value, including any backward-compatibility shims. +These methods previously delegated to namespace equivalents with a DeprecationWarning. +In 1.0 GA they are fully removed; each call now raises AttributeError. +Callers must use the operation namespaces directly (records.*, query.*, tables.*, files.*). """ import unittest @@ -18,281 +17,95 @@ class TestClientDeprecations(unittest.TestCase): - """Verify every deprecated flat method warns and delegates correctly.""" + """All formerly-deprecated flat methods are now removed and raise AttributeError.""" def setUp(self): - """Set up test fixtures before each test method.""" self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - # Mock the internal OData client so namespace methods resolve without - # making real HTTP calls. - self.client._odata = MagicMock() - # ---------------------------------------------------------------- create + # ---------------------------------------------------------------- records - def test_create_warns(self): - """client.create() emits a DeprecationWarning.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): + def test_create_removed(self): + """client.create() → use client.records.create()""" + with self.assertRaises(AttributeError): self.client.create("account", {"name": "Test"}) def test_create_single_returns_list(self): - """client.create() wraps a single GUID in a list for backward compat. - - records.create() returns a bare string for a single dict, but the - deprecated client.create() always returned list[str]. - """ - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create.return_value = "guid-123" - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", {"name": "A"}) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-123"]) + """client.create() single-dict shim is gone; client.records.create() returns str.""" + with self.assertRaises(AttributeError): + self.client.create("account", {"name": "A"}) def test_create_bulk_returns_list(self): - """client.create() with a list payload returns list[str] directly.""" - self.client._odata._entity_set_from_schema_name.return_value = "accounts" - self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create("account", [{"name": "A"}, {"name": "B"}]) - - self.assertIsInstance(result, list) - self.assertEqual(result, ["guid-1", "guid-2"]) - - # ---------------------------------------------------------------- update + """client.create() list-payload shim is gone; client.records.create() returns list[str].""" + with self.assertRaises(AttributeError): + self.client.create("account", [{"name": "A"}, {"name": "B"}]) def test_update_warns_and_delegates(self): - """client.update() emits a DeprecationWarning and delegates to records.update.""" - with self.assertWarns(DeprecationWarning): - self.client.update( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - self.client._odata._update.assert_called_once_with( - "account", - "00000000-0000-0000-0000-000000000001", - {"telephone1": "555-0199"}, - ) - - # ---------------------------------------------------------------- delete + """client.update() → use client.records.update()""" + with self.assertRaises(AttributeError): + self.client.update("account", "guid-1", {"telephone1": "555-0199"}) def test_delete_warns_and_delegates(self): - """client.delete() emits a DeprecationWarning and delegates to records.delete.""" - with self.assertWarns(DeprecationWarning): - self.client.delete("account", "00000000-0000-0000-0000-000000000001") - - self.client._odata._delete.assert_called_once_with("account", "00000000-0000-0000-0000-000000000001") - - # ------------------------------------------------------------------- get + """client.delete() → use client.records.delete()""" + with self.assertRaises(AttributeError): + self.client.delete("account", "guid-1") def test_get_single_warns(self): - """client.get() with record_id emits a DeprecationWarning and delegates - to records.get. - """ - expected = {"accountid": "guid-1", "name": "Contoso"} - self.client._odata._get.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", record_id="guid-1") - - self.client._odata._get.assert_called_once_with("account", "guid-1", select=None) - self.assertEqual(result["accountid"], "guid-1") - self.assertEqual(result["name"], "Contoso") + """client.get(record_id=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", record_id="guid-1") def test_get_multiple_warns(self): - """client.get() without record_id emits a DeprecationWarning and delegates - to records.get. - """ - page = [{"accountid": "1", "name": "A"}, {"accountid": "2", "name": "B"}] - self.client._odata._get_multiple.return_value = iter([page]) - - with self.assertWarns(DeprecationWarning): - result = self.client.get("account", filter="statecode eq 0", top=10) + """client.get(filter=...) → use client.records.get()""" + with self.assertRaises(AttributeError): + self.client.get("account", filter="statecode eq 0", top=10) - # The result is a generator; consume it. - pages = list(result) - self.assertEqual(len(pages), 1) - self.assertEqual(pages[0][0]["name"], "A") - self.assertEqual(pages[0][1]["name"], "B") - - self.client._odata._get_multiple.assert_called_once_with( - "account", - select=None, - filter="statecode eq 0", - orderby=None, - top=10, - expand=None, - page_size=None, - count=False, - include_annotations=None, - ) - - # ------------------------------------------------------------- query_sql + # ----------------------------------------------------------------- query def test_query_sql_warns(self): - """client.query_sql() emits a DeprecationWarning and delegates to - query.sql. - """ - expected_rows = [{"name": "Contoso"}, {"name": "Fabrikam"}] - self.client._odata._query_sql.return_value = expected_rows - - with self.assertWarns(DeprecationWarning): - result = self.client.query_sql("SELECT name FROM account") - - self.client._odata._query_sql.assert_called_once_with("SELECT name FROM account") - self.assertEqual(len(result), 2) - self.assertEqual(result[0]["name"], "Contoso") - self.assertEqual(result[1]["name"], "Fabrikam") + """client.query_sql() → use client.query.sql()""" + with self.assertRaises(AttributeError): + self.client.query_sql("SELECT name FROM account") - # -------------------------------------------------------- get_table_info + # --------------------------------------------------------------- tables def test_get_table_info_warns(self): - """client.get_table_info() emits a DeprecationWarning and delegates to - tables.get. - """ - expected_info = { - "table_schema_name": "new_MyTable", - "table_logical_name": "new_mytable", - "entity_set_name": "new_mytables", - "metadata_id": "meta-guid", - } - self.client._odata._get_table_info.return_value = expected_info - - with self.assertWarns(DeprecationWarning): - result = self.client.get_table_info("new_MyTable") - - self.client._odata._get_table_info.assert_called_once_with("new_MyTable") - self.assertEqual(result["table_schema_name"], "new_MyTable") - self.assertEqual(result["entity_set_name"], "new_mytables") - - # --------------------------------------------------------- create_table + """client.get_table_info() → use client.tables.get()""" + with self.assertRaises(AttributeError): + self.client.get_table_info("new_MyTable") def test_create_table_warns(self): - """client.create_table() emits a DeprecationWarning and maps legacy - parameter names (solution_unique_name -> solution, - primary_column_schema_name -> primary_column) when delegating to - tables.create. - """ - expected = { - "table_schema_name": "new_Product", - "entity_set_name": "new_products", - "table_logical_name": "new_product", - "metadata_id": "meta-guid", - "columns_created": ["new_Price"], - } - self.client._odata._create_table.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.create_table( - "new_Product", - {"new_Price": "decimal"}, - solution_unique_name="MySolution", - primary_column_schema_name="new_ProductName", - ) - - # Verify that the internal _create_table received the mapped params. - self.client._odata._create_table.assert_called_once_with( - "new_Product", - {"new_Price": "decimal"}, - "MySolution", - "new_ProductName", - None, - ) - self.assertEqual(result["table_schema_name"], "new_Product") - self.assertEqual(result["columns_created"], ["new_Price"]) - - # --------------------------------------------------------- delete_table + """client.create_table() → use client.tables.create()""" + with self.assertRaises(AttributeError): + self.client.create_table("new_Product", {"new_Price": "decimal"}) def test_delete_table_warns(self): - """client.delete_table() emits a DeprecationWarning and delegates to - tables.delete. - """ - with self.assertWarns(DeprecationWarning): + """client.delete_table() → use client.tables.delete()""" + with self.assertRaises(AttributeError): self.client.delete_table("new_MyTestTable") - self.client._odata._delete_table.assert_called_once_with("new_MyTestTable") - - # ---------------------------------------------------------- list_tables - def test_list_tables_warns(self): - """client.list_tables() emits a DeprecationWarning and delegates to - tables.list. - """ - expected = [{"LogicalName": "account"}, {"LogicalName": "contact"}] - self.client._odata._list_tables.return_value = expected - - with self.assertWarns(DeprecationWarning): - result = self.client.list_tables() - - self.client._odata._list_tables.assert_called_once() - self.assertEqual(result, expected) - - # ------------------------------------------------------- create_columns + """client.list_tables() → use client.tables.list()""" + with self.assertRaises(AttributeError): + self.client.list_tables() def test_create_columns_warns(self): - """client.create_columns() emits a DeprecationWarning and delegates to - tables.add_columns. - """ - self.client._odata._create_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.create_columns( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - - self.client._odata._create_columns.assert_called_once_with( - "new_MyTestTable", - {"new_Notes": "string", "new_Active": "bool"}, - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ------------------------------------------------------- delete_columns + """client.create_columns() → use client.tables.add_columns()""" + with self.assertRaises(AttributeError): + self.client.create_columns("new_MyTestTable", {"new_Notes": "string"}) def test_delete_columns_warns(self): - """client.delete_columns() emits a DeprecationWarning and delegates to - tables.remove_columns. - """ - self.client._odata._delete_columns.return_value = ["new_Notes", "new_Active"] - - with self.assertWarns(DeprecationWarning): - result = self.client.delete_columns( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) + """client.delete_columns() → use client.tables.remove_columns()""" + with self.assertRaises(AttributeError): + self.client.delete_columns("new_MyTestTable", ["new_Notes"]) - self.client._odata._delete_columns.assert_called_once_with( - "new_MyTestTable", - ["new_Notes", "new_Active"], - ) - self.assertEqual(result, ["new_Notes", "new_Active"]) - - # ----------------------------------------------------------- upload_file + # ----------------------------------------------------------------- files def test_upload_file_warns(self): - """client.upload_file() emits a DeprecationWarning and delegates - to files.upload. - """ - with self.assertWarns(DeprecationWarning): + """client.upload_file() → use client.files.upload()""" + with self.assertRaises(AttributeError): self.client.upload_file("account", "guid-1", "new_Document", "/path/to/file.pdf") - self.client._odata._upload_file.assert_called_once_with( - "account", - "guid-1", - "new_Document", - "/path/to/file.pdf", - mode=None, - mime_type=None, - if_none_match=True, - ) - if __name__ == "__main__": unittest.main() diff --git a/tests/unit/test_migration_tool.py b/tests/unit/test_migration_tool.py new file mode 100644 index 00000000..88173a8f --- /dev/null +++ b/tests/unit/test_migration_tool.py @@ -0,0 +1,303 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Unit tests for tools/migrate_v0_to_v1.py. + +Covers: +- QueryBuilder.to_dataframe() -> .execute().to_dataframe() (auto-rewrite) +- QueryResult.to_dataframe() left untouched (receiver is .execute()) +- QueryBuilder chain via .select(), .where(), .filter_eq() all trigger the rewrite +- client.get(t, id) -> client.records.get(t, id) (top-level shortcut) +- batch.records.get(t, id) -> batch.records.retrieve(t, id) +- .filter_eq / .filter_ne / .filter_gt -> .where(col(...) OP v) +- .filter_null / .filter_not_null -> .where(col(...).is_null/is_not_null()) +- .filter_raw / .filter -> .where(raw(...)) +- .execute(by_page=True) -> .execute_pages() +- .execute(by_page=False) -> .execute() with flag stripped +- find_manual_patterns: flags client.records.get(), execute(by_page=variable), client.dataframe.get() +""" + +import textwrap +import unittest + +try: + import libcst # noqa: F401 + + _LIBCST_AVAILABLE = True +except ImportError: + _LIBCST_AVAILABLE = False + +_skip_no_libcst = unittest.skipUnless(_LIBCST_AVAILABLE, "libcst not installed") + + +def _migrate(source: str, *, client_var: str = "client") -> str: + from tools.migrate_v0_to_v1 import migrate_source + + return migrate_source(textwrap.dedent(source), client_var=client_var) + + +def _find_manual(source: str, *, client_var: str = "client") -> list: + from tools.migrate_v0_to_v1 import find_manual_patterns + + return find_manual_patterns(textwrap.dedent(source), client_var=client_var) + + +# --------------------------------------------------------------------------- +# QueryBuilder.to_dataframe() -> .execute().to_dataframe() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestToDataframeRewrite(unittest.TestCase): + """QueryBuilder.to_dataframe() receives .execute() insertion.""" + + def test_builder_chain_gets_execute_inserted(self): + src = "df = client.query.builder('account').select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + self.assertNotIn(".to_dataframe().to_dataframe()", out) + + def test_where_chain_triggers_rewrite(self): + src = "df = q.where(col('statecode') == 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_filter_eq_chain_triggers_rewrite(self): + src = "df = q.filter_eq('statecode', 0).to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_select_alone_triggers_rewrite(self): + src = "df = q.select('name', 'revenue').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + def test_already_executed_not_double_wrapped(self): + src = "df = q.select('name').execute().to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute().execute()", out) + self.assertIn(".execute().to_dataframe()", out) + + def test_unrelated_to_dataframe_not_rewritten(self): + src = "df = some_result.to_dataframe()\n" + out = _migrate(src) + self.assertNotIn(".execute()", out) + self.assertIn("some_result.to_dataframe()", out) + + def test_full_chain_structure_preserved(self): + src = "df = client.query.builder('account')\\\n" " .select('name')\\\n" " .to_dataframe()\n" + out = _migrate(src) + # .execute() is inserted before .to_dataframe(); a line-continuation may separate them + self.assertIn(".execute()", out) + self.assertIn(".to_dataframe()", out) + self.assertNotIn(".get(", out) + + def test_rewrite_inside_assignment(self): + src = "result = builder.select('name').to_dataframe()\n" + out = _migrate(src) + self.assertIn(".execute().to_dataframe()", out) + + +# --------------------------------------------------------------------------- +# Top-level shortcut rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestClientShortcutRewrites(unittest.TestCase): + def test_client_get_becomes_records_get(self): + src = "r = client.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.get(", out) + + def test_client_create_becomes_records_create(self): + src = "client.create('account', {'name': 'X'})\n" + out = _migrate(src) + self.assertIn("client.records.create(", out) + + def test_client_delete_becomes_records_delete(self): + src = "client.delete('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.delete(", out) + + def test_client_update_becomes_records_update(self): + src = "client.update('account', 'abc', {'name': 'Y'})\n" + out = _migrate(src) + self.assertIn("client.records.update(", out) + + def test_client_query_sql_becomes_query_sql(self): + src = "rows = client.query_sql('SELECT * FROM account')\n" + out = _migrate(src) + self.assertIn("client.query.sql(", out) + + def test_client_get_table_info_becomes_tables_get(self): + src = "info = client.get_table_info('account')\n" + out = _migrate(src) + self.assertIn("client.tables.get(", out) + + def test_client_list_tables_becomes_tables_list(self): + src = "tables = client.list_tables()\n" + out = _migrate(src) + self.assertIn("client.tables.list(", out) + + def test_client_var_override(self): + src = "r = svc.get('account', 'abc')\n" + out = _migrate(src, client_var="svc") + self.assertIn("svc.records.get(", out) + + def test_client_get_not_matched_on_other_receiver(self): + src = "v = record.get('name')\n" + out = _migrate(src) + self.assertIn("record.get(", out) + self.assertNotIn("record.records.get(", out) + + +# --------------------------------------------------------------------------- +# batch.records.get() -> batch.records.retrieve() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestBatchRecordsGetRewrite(unittest.TestCase): + def test_batch_records_get_becomes_retrieve(self): + src = "batch.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("batch.records.retrieve(", out) + self.assertNotIn("batch.records.get(", out) + + def test_client_records_get_not_rewritten(self): + src = "client.records.get('account', 'abc')\n" + out = _migrate(src) + self.assertIn("client.records.get(", out) + self.assertNotIn("client.records.retrieve(", out) + + +# --------------------------------------------------------------------------- +# .filter_*() -> .where(col(...) ...) rewrites +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFilterMethodRewrites(unittest.TestCase): + def test_filter_eq(self): + src = "q.filter_eq('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + self.assertIn("col(", out) + + def test_filter_ne(self): + src = "q.filter_ne('statecode', 0)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_gt(self): + src = "q.filter_gt('revenue', 1000)\n" + out = _migrate(src) + self.assertIn(".where(", out) + + def test_filter_null(self): + src = "q.filter_null('email')\n" + out = _migrate(src) + self.assertIn(".is_null()", out) + + def test_filter_not_null(self): + src = "q.filter_not_null('email')\n" + out = _migrate(src) + self.assertIn(".is_not_null()", out) + + def test_filter_raw(self): + src = "q.filter_raw('statecode eq 0')\n" + out = _migrate(src) + self.assertIn("raw(", out) + + def test_filter_string_literal(self): + src = "q.filter('statecode eq 0')\n" + out = _migrate(src) + self.assertIn(".where(raw(", out) + + def test_filter_between(self): + src = "q.filter_between('revenue', 1000, 5000)\n" + out = _migrate(src) + self.assertIn(".between(", out) + + def test_filter_in(self): + src = "q.filter_in('statecode', [0, 1])\n" + out = _migrate(src) + self.assertIn(".in_(", out) + + +# --------------------------------------------------------------------------- +# .execute(by_page=...) -> .execute_pages() / .execute() +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestExecuteByPageRewrite(unittest.TestCase): + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = q.execute(by_page=True)\n" + out = _migrate(src) + self.assertIn(".execute_pages()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_false_strips_flag(self): + src = "result = q.execute(by_page=False)\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute_pages", out) + + def test_execute_no_args_unchanged(self): + src = "result = q.execute()\n" + out = _migrate(src) + self.assertIn(".execute()", out) + self.assertNotIn("execute_pages", out) + + +# --------------------------------------------------------------------------- +# find_manual_patterns +# --------------------------------------------------------------------------- + + +@_skip_no_libcst +class TestFindManualPatterns(unittest.TestCase): + def test_client_records_get_flagged(self): + src = "client.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "q.execute(by_page=flag)\n" + findings = _find_manual(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + src = "q.execute(by_page=True)\n" + findings = _find_manual(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_client_dataframe_get_flagged(self): + src = "client.dataframe.get('account')\n" + findings = _find_manual(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_query_sql_select_flagged(self): + src = "client.query.sql_select('account', ['name'])\n" + findings = _find_manual(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_clean_code_has_no_findings(self): + src = ( + "result = client.records.retrieve('account', 'abc')\n" "pages = client.records.list('account').execute()\n" + ) + findings = _find_manual(src) + self.assertEqual(findings, []) + + def test_batch_records_get_not_flagged(self): + src = "batch.records.get('account', 'abc')\n" + findings = _find_manual(src) + self.assertFalse(any("records.get" in f for f in findings)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase1_ga.py b/tests/unit/test_phase1_ga.py new file mode 100644 index 00000000..8069ffd1 --- /dev/null +++ b/tests/unit/test_phase1_ga.py @@ -0,0 +1,859 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Comprehensive Phase 1 GA regression tests. + +Verifies all Phase 1 breaking changes and deprecations in one place: + +1. All 12 deprecated client flat methods raise AttributeError (removed). +2. All 15 deprecated filter factory functions emit DeprecationWarning on CALL + (not on import). +3. GA filter functions col() and raw() emit NO deprecation warning. +4. dataframe.get() emits DeprecationWarning on call. +5. All deprecated factories remain functional (correct OData output). +6. All 16 filter_* builder methods raise AttributeError on QueryBuilder (removed). +7. No DeprecationWarning is emitted at module import time. +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.client import DataverseClient +from PowerPlatform.Dataverse.models.query_builder import QueryBuilder + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_client(): + """Create a DataverseClient with a mock credential and mock _odata.""" + credential = MagicMock(spec=TokenCredential) + client = DataverseClient("https://example.crm.dynamics.com", credential) + client._odata = MagicMock() + return client + + +def _catch_warnings(*categories): + """Context manager: catch all warnings, return the list.""" + return warnings.catch_warnings(record=True) + + +# --------------------------------------------------------------------------- +# 1. Removed client flat methods raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedClientFlatMethods(unittest.TestCase): + """All 12 formerly-deprecated client flat methods are removed in GA.""" + + def setUp(self): + self.client = _make_client() + + def _assert_removed(self, method_name, *args, **kwargs): + with self.assertRaises(AttributeError, msg=f"client.{method_name} should not exist"): + getattr(self.client, method_name)(*args, **kwargs) + + def test_create_removed(self): + self._assert_removed("create", "account", {"name": "Test"}) + + def test_update_removed(self): + self._assert_removed("update", "account", "guid-1", {"name": "Test"}) + + def test_delete_removed(self): + self._assert_removed("delete", "account", "guid-1") + + def test_get_removed(self): + self._assert_removed("get", "account", "guid-1") + + def test_query_sql_removed(self): + self._assert_removed("query_sql", "SELECT name FROM account") + + def test_get_table_info_removed(self): + self._assert_removed("get_table_info", "account") + + def test_create_table_removed(self): + self._assert_removed("create_table", "new_Test", {}) + + def test_delete_table_removed(self): + self._assert_removed("delete_table", "new_Test") + + def test_list_tables_removed(self): + self._assert_removed("list_tables") + + def test_create_columns_removed(self): + self._assert_removed("create_columns", "account", {}) + + def test_delete_columns_removed(self): + self._assert_removed("delete_columns", "account", []) + + def test_upload_file_removed(self): + self._assert_removed("upload_file", "account", "guid-1", "col", "/path") + + +# --------------------------------------------------------------------------- +# 2. Deprecated filter factories emit DeprecationWarning on CALL (not import) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFilterFactoriesWarnOnCall(unittest.TestCase): + """All 15 deprecated filter factories emit DeprecationWarning when called. + + The warning must fire on CALL, not on import — importing the module must + be warning-free. + """ + + def _assert_warns_on_call(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, f"{func.__name__}() did not emit DeprecationWarning") + return result, dep_warnings + + def _assert_single_warning(self, func, *args, **kwargs): + """Verify exactly one DeprecationWarning is emitted (no chained warnings).""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 1, f"{func.__name__}() emitted {len(dep_warnings)} warnings (expected exactly 1)" + ) + + def test_no_warning_on_import(self): + """Importing the filters module emits no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + import importlib + import PowerPlatform.Dataverse.models.filters as _f + + importlib.reload(_f) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "Import must not emit DeprecationWarning") + + def test_eq_warns(self): + from PowerPlatform.Dataverse.models.filters import eq + + self._assert_warns_on_call(eq, "name", "Contoso") + + def test_ne_warns(self): + from PowerPlatform.Dataverse.models.filters import ne + + self._assert_warns_on_call(ne, "statecode", 1) + + def test_gt_warns(self): + from PowerPlatform.Dataverse.models.filters import gt + + self._assert_warns_on_call(gt, "revenue", 1000000) + + def test_ge_warns(self): + from PowerPlatform.Dataverse.models.filters import ge + + self._assert_warns_on_call(ge, "revenue", 1000000) + + def test_lt_warns(self): + from PowerPlatform.Dataverse.models.filters import lt + + self._assert_warns_on_call(lt, "revenue", 500000) + + def test_le_warns(self): + from PowerPlatform.Dataverse.models.filters import le + + self._assert_warns_on_call(le, "revenue", 500000) + + def test_contains_warns(self): + from PowerPlatform.Dataverse.models.filters import contains + + self._assert_warns_on_call(contains, "name", "Corp") + + def test_startswith_warns(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self._assert_warns_on_call(startswith, "name", "Con") + + def test_endswith_warns(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self._assert_warns_on_call(endswith, "name", "Ltd") + + def test_between_warns(self): + from PowerPlatform.Dataverse.models.filters import between + + self._assert_warns_on_call(between, "revenue", 100000, 500000) + + def test_is_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self._assert_warns_on_call(is_null, "telephone1") + + def test_is_not_null_warns(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self._assert_warns_on_call(is_not_null, "telephone1") + + def test_filter_in_warns(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self._assert_warns_on_call(filter_in, "statecode", [0, 1]) + + def test_not_in_warns(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self._assert_warns_on_call(not_in, "statecode", [2, 3]) + + def test_not_between_warns(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_warns_on_call(not_between, "revenue", 100000, 500000) + + def test_between_emits_exactly_one_warning(self): + """between() must not chain through deprecated ge/le (would emit 3 warnings).""" + from PowerPlatform.Dataverse.models.filters import between + + self._assert_single_warning(between, "revenue", 100000, 500000) + + def test_not_between_emits_exactly_one_warning(self): + """not_between() must not chain through deprecated ge/le.""" + from PowerPlatform.Dataverse.models.filters import not_between + + self._assert_single_warning(not_between, "revenue", 100000, 500000) + + def test_warning_is_deprecation_warning_class(self): + """Each factory's warning category must be DeprecationWarning, not its subclass.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + w = caught[0] + self.assertIs(w.category, DeprecationWarning) + + def test_warning_message_names_replacement(self): + """Each warning message should name the col()-based replacement.""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + eq("name", "Test") + self.assertIn("col(", str(caught[0].message)) + + +# --------------------------------------------------------------------------- +# 3. GA functions col() and raw() emit NO deprecation warning +# --------------------------------------------------------------------------- + + +class TestGAFunctionsNoWarning(unittest.TestCase): + """col() and raw() are GA — must never emit DeprecationWarning.""" + + def _assert_no_dep_warning(self, func, *args, **kwargs): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + func(*args, **kwargs) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual( + len(dep_warnings), 0, f"{func.__name__}() emitted unexpected DeprecationWarning: {dep_warnings}" + ) + + def test_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + self._assert_no_dep_warning(col, "statecode") + + def test_raw_no_warning(self): + from PowerPlatform.Dataverse.models.filters import raw + + self._assert_no_dep_warning(raw, "statecode eq 0") + + def test_col_eq_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode") == 0 + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_comparison_chain_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = (col("statecode") == 0) & (col("revenue") > 100000) + _ = expr.to_odata() + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_between_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("revenue").not_between(100000, 500000) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").in_([0, 1]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_in_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("statecode").not_in([2, 3]) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").like("Contoso%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_col_not_like_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + expr = col("name").not_like("%Corp%") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + def test_where_with_col_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + QueryBuilder("account").where(col("statecode") == 0) + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + + +# --------------------------------------------------------------------------- +# 4. dataframe.get() emits DeprecationWarning on call +# --------------------------------------------------------------------------- + + +class TestDataframeGetDeprecation(unittest.TestCase): + """dataframe.get() must emit DeprecationWarning on call (not on import).""" + + def setUp(self): + self.client = _make_client() + # Set up _odata so records.get() can be called + self.client._odata._get_multiple.return_value = iter([]) + self.client._odata._get.return_value = MagicMock(data={}) + + def test_dataframe_get_warns_on_call(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertGreater(len(dep_warnings), 0, "dataframe.get() did not emit DeprecationWarning") + + def test_dataframe_get_warning_message(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + self.client.dataframe.get("account", select=["name"], top=10) + except Exception: + pass + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + msg = str(dep_warnings[0].message) + self.assertIn("dataframe.get()", msg) + self.assertIn("builder", msg) + + def test_dataframe_other_methods_no_warning(self): + """dataframe.sql(), dataframe.create(), etc. must NOT warn.""" + + self.client._odata._query_sql.return_value = [] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.dataframe.sql("SELECT name FROM account") + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, "dataframe.sql() must not warn") + + +# --------------------------------------------------------------------------- +# 5. Deprecated factories remain functional (correct OData output) +# --------------------------------------------------------------------------- + + +class TestDeprecatedFactoriesStillFunctional(unittest.TestCase): + """Despite the warning, deprecated factories produce correct OData strings.""" + + def _odata(self, func, *args): + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + return func(*args).to_odata() + + def test_eq_functional(self): + from PowerPlatform.Dataverse.models.filters import eq + + self.assertEqual(self._odata(eq, "name", "Contoso"), "name eq 'Contoso'") + + def test_ne_functional(self): + from PowerPlatform.Dataverse.models.filters import ne + + self.assertEqual(self._odata(ne, "statecode", 1), "statecode ne 1") + + def test_gt_functional(self): + from PowerPlatform.Dataverse.models.filters import gt + + self.assertEqual(self._odata(gt, "revenue", 1000000), "revenue gt 1000000") + + def test_ge_functional(self): + from PowerPlatform.Dataverse.models.filters import ge + + self.assertEqual(self._odata(ge, "revenue", 1000000), "revenue ge 1000000") + + def test_lt_functional(self): + from PowerPlatform.Dataverse.models.filters import lt + + self.assertEqual(self._odata(lt, "revenue", 500000), "revenue lt 500000") + + def test_le_functional(self): + from PowerPlatform.Dataverse.models.filters import le + + self.assertEqual(self._odata(le, "revenue", 500000), "revenue le 500000") + + def test_contains_functional(self): + from PowerPlatform.Dataverse.models.filters import contains + + self.assertEqual(self._odata(contains, "name", "Corp"), "contains(name, 'Corp')") + + def test_startswith_functional(self): + from PowerPlatform.Dataverse.models.filters import startswith + + self.assertEqual(self._odata(startswith, "name", "Con"), "startswith(name, 'Con')") + + def test_endswith_functional(self): + from PowerPlatform.Dataverse.models.filters import endswith + + self.assertEqual(self._odata(endswith, "name", "Ltd"), "endswith(name, 'Ltd')") + + def test_between_functional(self): + from PowerPlatform.Dataverse.models.filters import between + + self.assertEqual( + self._odata(between, "revenue", 100000, 500000), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_is_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_null + + self.assertEqual(self._odata(is_null, "telephone1"), "telephone1 eq null") + + def test_is_not_null_functional(self): + from PowerPlatform.Dataverse.models.filters import is_not_null + + self.assertEqual(self._odata(is_not_null, "telephone1"), "telephone1 ne null") + + def test_filter_in_functional(self): + from PowerPlatform.Dataverse.models.filters import filter_in + + self.assertEqual( + self._odata(filter_in, "statecode", [0, 1]), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in_functional(self): + from PowerPlatform.Dataverse.models.filters import not_in + + self.assertEqual( + self._odata(not_in, "statecode", [2, 3]), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_not_between_functional(self): + from PowerPlatform.Dataverse.models.filters import not_between + + self.assertEqual( + self._odata(not_between, "revenue", 100000, 500000), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_deprecated_factory_usable_in_where(self): + """Deprecated factories still produce valid FilterExpression for where().""" + from PowerPlatform.Dataverse.models.filters import eq + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = eq("statecode", 0) + qb = QueryBuilder("account").where(expr) + self.assertEqual(qb.build()["filter"], "statecode eq 0") + + def test_deprecated_factories_composable(self): + """Deprecated factories still compose correctly with & and |.""" + from PowerPlatform.Dataverse.models.filters import eq, gt + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + expr = (eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000) + self.assertEqual( + expr.to_odata(), + "((statecode eq 0 or statecode eq 1) and revenue gt 100000)", + ) + + +# --------------------------------------------------------------------------- +# 6. All 16 filter_* builder methods raise AttributeError on QueryBuilder +# --------------------------------------------------------------------------- + + +class TestRemovedBuilderFilterMethods(unittest.TestCase): + """All 16 filter_* methods were removed from QueryBuilder in GA.""" + + def setUp(self): + self.qb = QueryBuilder("account") + + def _assert_removed(self, method_name, *args): + with self.assertRaises(AttributeError, msg=f"QueryBuilder.{method_name} should not exist"): + getattr(self.qb, method_name)(*args) + + def test_filter_eq_removed(self): + self._assert_removed("filter_eq", "name", "Contoso") + + def test_filter_ne_removed(self): + self._assert_removed("filter_ne", "statecode", 1) + + def test_filter_gt_removed(self): + self._assert_removed("filter_gt", "revenue", 0) + + def test_filter_ge_removed(self): + self._assert_removed("filter_ge", "revenue", 0) + + def test_filter_lt_removed(self): + self._assert_removed("filter_lt", "revenue", 0) + + def test_filter_le_removed(self): + self._assert_removed("filter_le", "revenue", 0) + + def test_filter_contains_removed(self): + self._assert_removed("filter_contains", "name", "Corp") + + def test_filter_startswith_removed(self): + self._assert_removed("filter_startswith", "name", "Con") + + def test_filter_endswith_removed(self): + self._assert_removed("filter_endswith", "name", "Ltd") + + def test_filter_null_removed(self): + self._assert_removed("filter_null", "telephone1") + + def test_filter_not_null_removed(self): + self._assert_removed("filter_not_null", "telephone1") + + def test_filter_in_removed(self): + self._assert_removed("filter_in", "statecode", [0, 1]) + + def test_filter_not_in_removed(self): + self._assert_removed("filter_not_in", "statecode", [0, 1]) + + def test_filter_between_removed(self): + self._assert_removed("filter_between", "revenue", 100, 500) + + def test_filter_not_between_removed(self): + self._assert_removed("filter_not_between", "revenue", 100, 500) + + def test_filter_raw_removed(self): + self._assert_removed("filter_raw", "statecode eq 0") + + +# --------------------------------------------------------------------------- +# 7. ColumnProxy (col()) correctness — all operators and methods +# --------------------------------------------------------------------------- + + +class TestColumnProxyOperators(unittest.TestCase): + """col() proxy covers all operators and methods correctly.""" + + def test_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("name") == "Contoso").to_odata(), "name eq 'Contoso'") + + def test_ne(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("statecode") != 1).to_odata(), "statecode ne 1") + + def test_gt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") > 1000000).to_odata(), "revenue gt 1000000") + + def test_ge(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") >= 1000000).to_odata(), "revenue ge 1000000") + + def test_lt(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") < 500000).to_odata(), "revenue lt 500000") + + def test_le(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("revenue") <= 500000).to_odata(), "revenue le 500000") + + def test_is_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_null().to_odata(), "telephone1 eq null") + + def test_is_not_null(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("telephone1").is_not_null().to_odata(), "telephone1 ne null") + + def test_in_(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").in_([0, 1]).to_odata(), + 'Microsoft.Dynamics.CRM.In(PropertyName=\'statecode\',PropertyValues=["0","1"])', + ) + + def test_not_in(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("statecode").not_in([2, 3]).to_odata(), + 'Microsoft.Dynamics.CRM.NotIn(PropertyName=\'statecode\',PropertyValues=["2","3"])', + ) + + def test_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").between(100000, 500000).to_odata(), + "(revenue ge 100000 and revenue le 500000)", + ) + + def test_not_between(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("revenue").not_between(100000, 500000).to_odata(), + "not ((revenue ge 100000 and revenue le 500000))", + ) + + def test_contains(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").contains("Corp").to_odata(), "contains(name, 'Corp')") + + def test_startswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").startswith("Con").to_odata(), "startswith(name, 'Con')") + + def test_endswith(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").endswith("Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso%").to_odata(), "startswith(name, 'Contoso')") + + def test_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Ltd").to_odata(), "endswith(name, 'Ltd')") + + def test_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("%Corp%").to_odata(), "contains(name, 'Corp')") + + def test_like_no_wildcard_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual(col("name").like("Contoso").to_odata(), "name eq 'Contoso'") + + def test_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").like("Con%oso") + + def test_not_like_startswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso%").to_odata(), + "not (startswith(name, 'Contoso'))", + ) + + def test_not_like_endswith_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Ltd").to_odata(), + "not (endswith(name, 'Ltd'))", + ) + + def test_not_like_contains_pattern(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("%Corp%").to_odata(), + "not (contains(name, 'Corp'))", + ) + + def test_not_like_no_wildcard_negated_equality(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual( + col("name").not_like("Contoso").to_odata(), + "not (name eq 'Contoso')", + ) + + def test_not_like_interior_wildcard_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("name").not_like("Con%oso") + + def test_column_name_lowercased(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("StateCode") == 0).to_odata(), "statecode eq 0") + + def test_empty_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("") + + def test_whitespace_column_name_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col(" ") + + def test_boolean_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("active") == True).to_odata(), "active eq true") # noqa: E712 + self.assertEqual((col("active") == False).to_odata(), "active eq false") # noqa: E712 + + def test_none_eq(self): + from PowerPlatform.Dataverse.models.filters import col + + self.assertEqual((col("telephone1") == None).to_odata(), "telephone1 eq null") # noqa: E711 + + def test_and_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) & (col("revenue") > 100000) + self.assertEqual(expr.to_odata(), "(statecode eq 0 and revenue gt 100000)") + + def test_or_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = (col("statecode") == 0) | (col("statecode") == 1) + self.assertEqual(expr.to_odata(), "(statecode eq 0 or statecode eq 1)") + + def test_not_composition(self): + from PowerPlatform.Dataverse.models.filters import col + + expr = ~(col("statecode") == 1) + self.assertEqual(expr.to_odata(), "not (statecode eq 1)") + + def test_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").in_([]) + + def test_not_in_empty_raises(self): + from PowerPlatform.Dataverse.models.filters import col + + with self.assertRaises(ValueError): + col("statecode").not_in([]) + + +# --------------------------------------------------------------------------- +# 8. GA namespace-level fluent builder (where + col) end-to-end +# --------------------------------------------------------------------------- + + +class TestGABuilderEndToEnd(unittest.TestCase): + """End-to-end verification of the GA query builder without any deprecated APIs.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_builder_with_col_exprs_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = ( + self.client.query.builder("account") + .select("name", "revenue") + .where((col("statecode") == 0) | (col("statecode") == 1)) + .where(col("revenue") > 100000) + .order_by("revenue", descending=True) + .top(50) + ) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0, f"GA API emitted warnings: {dep_warnings}") + self.assertEqual( + params["filter"], + "(statecode eq 0 or statecode eq 1) and revenue gt 100000", + ) + self.assertEqual(params["select"], ["name", "revenue"]) + self.assertEqual(params["top"], 50) + + def test_builder_with_raw_no_warnings(self): + from PowerPlatform.Dataverse.models.filters import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + qb = self.client.query.builder("account").select("name").where(raw("statecode eq 0")).top(10) + params = qb.build() + + dep_warnings = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(dep_warnings), 0) + self.assertEqual(params["filter"], "statecode eq 0") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase2_ga.py b/tests/unit/test_phase2_ga.py new file mode 100644 index 00000000..0850a780 --- /dev/null +++ b/tests/unit/test_phase2_ga.py @@ -0,0 +1,445 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 2 GA regression tests. + +Covers: +- QueryResult class (models/record.py) +- execute() returns QueryResult in flat mode +- execute(by_page=True) still returns Iterable[list[Record]] +- col, raw, QueryResult re-exports from models/__init__ and package root +- pyproject.toml migration optional dep +""" + +import unittest +import warnings +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult, Record + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + return client + + +class TestQueryResultClass(unittest.TestCase): + """Unit tests for QueryResult.""" + + def _records(self, n=3): + return [Record(id=f"id-{i}", table="account", data={"name": f"R{i}"}) for i in range(n)] + + # ----- construction / dunder + + def test_init_stores_records(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertIs(qr.records, recs) + + def test_empty_result(self): + qr = QueryResult([]) + self.assertEqual(len(qr), 0) + self.assertFalse(bool(qr)) + self.assertIsNone(qr.first()) + + def test_len(self): + self.assertEqual(len(QueryResult(self._records(5))), 5) + + def test_bool_true_when_nonempty(self): + self.assertTrue(bool(QueryResult(self._records(1)))) + + def test_bool_false_when_empty(self): + self.assertFalse(bool(QueryResult([]))) + + def test_iter_yields_records(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + def test_iter_multiple_times(self): + recs = self._records(2) + qr = QueryResult(recs) + self.assertEqual(list(qr), list(qr)) + + def test_repr_contains_count(self): + qr = QueryResult(self._records(7)) + self.assertIn("7", repr(qr)) + + # ----- first() + + def test_first_returns_first_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr.first(), recs[0]) + + def test_first_returns_none_when_empty(self): + self.assertIsNone(QueryResult([]).first()) + + # ----- __getitem__ + + def test_getitem_int_returns_record(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[0], recs[0]) + self.assertIs(qr[2], recs[2]) + + def test_getitem_negative_index(self): + recs = self._records(3) + qr = QueryResult(recs) + self.assertIs(qr[-1], recs[-1]) + + def test_getitem_out_of_range_raises(self): + qr = QueryResult(self._records(2)) + with self.assertRaises(IndexError): + _ = qr[99] + + def test_getitem_slice_returns_query_result(self): + recs = self._records(5) + qr = QueryResult(recs) + sliced = qr[1:3] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(list(sliced), recs[1:3]) + + def test_getitem_slice_empty(self): + qr = QueryResult(self._records(3)) + sliced = qr[10:] + self.assertIsInstance(sliced, QueryResult) + self.assertEqual(len(sliced), 0) + + # ----- to_dataframe() + + def test_to_dataframe_nonempty(self): + import pandas as pd + + recs = [ + Record(id="id-1", table="account", data={"name": "Contoso", "revenue": 1000}), + Record(id="id-2", table="account", data={"name": "Fabrikam", "revenue": 2000}), + ] + qr = QueryResult(recs) + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + self.assertIn("name", df.columns) + self.assertIn("revenue", df.columns) + self.assertEqual(df.iloc[0]["name"], "Contoso") + self.assertEqual(df.iloc[1]["revenue"], 2000) + + def test_to_dataframe_empty_returns_empty_df(self): + import pandas as pd + + df = QueryResult([]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + def test_to_dataframe_handles_plain_dicts(self): + """QueryResult.to_dataframe() handles plain dicts (no .data attribute).""" + import pandas as pd + + qr = QueryResult([{"name": "A"}, {"name": "B"}]) # type: ignore[arg-type] + df = qr.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + # ----- for r in result (backward compat) + + def test_backward_compat_for_loop(self): + recs = self._records(3) + qr = QueryResult(recs) + collected = [] + for r in qr: + collected.append(r) + self.assertEqual(collected, recs) + + def test_list_conversion(self): + recs = self._records(4) + qr = QueryResult(recs) + self.assertEqual(list(qr), recs) + + +class TestExecuteReturnsQueryResult(unittest.TestCase): + """execute() flat mode returns QueryResult.""" + + def setUp(self): + self.client = _make_client() + + def test_execute_flat_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + + def test_execute_flat_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + self.assertEqual(len(result), 3) + + def test_execute_flat_records_accessible(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "abc"}], + ] + ) + result = self.client.query.builder("account").select("name").execute() + first = result.first() + self.assertIsNotNone(first) + self.assertEqual(first["name"], "Contoso") + + def test_execute_flat_for_loop_backward_compat(self): + """for r in execute() still works — backward-compatible iteration.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + records = [] + for r in self.client.query.builder("account").select("name").execute(): + records.append(r) + self.assertEqual(len(records), 2) + + def test_execute_flat_list_backward_compat(self): + """list(execute()) still works.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "x"}], + ] + ) + records = list(self.client.query.builder("account").select("name").execute()) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_execute_by_page_not_query_result(self): + """execute(by_page=True) still returns page iterator, not QueryResult.""" + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}], + ] + ) + result = self.client.query.builder("account").select("name").execute(by_page=True) + self.assertNotIsInstance(result, QueryResult) + pages = list(result) + self.assertEqual(len(pages), 2) + + def test_execute_empty_returns_empty_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.query.builder("account").select("name").execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(bool(result)) + self.assertIsNone(result.first()) + + def test_execute_result_to_dataframe(self): + """execute().to_dataframe() works end-to-end.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "1"}, {"name": "Fabrikam", "accountid": "2"}], + ] + ) + df = self.client.query.builder("account").select("name").execute().to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_execute_no_deprecation_warnings(self): + """execute() flat mode emits no DeprecationWarnings.""" + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + (self.client.query.builder("account").select("name").where(col("statecode") == 0).execute()) + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + +class TestQueryBuilderToDataframe(unittest.TestCase): + """to_dataframe() delegates to QueryResult.to_dataframe() after execute().""" + + def setUp(self): + self.client = _make_client() + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 1) + + def test_to_dataframe_empty_preserves_select_columns(self): + """to_dataframe() on empty result keeps column names from select().""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").select("name", "revenue").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + self.assertListEqual(list(df.columns), ["name", "revenue"]) + + def test_to_dataframe_empty_no_select(self): + """to_dataframe() on empty result with no select() returns bare empty DataFrame.""" + import pandas as pd + + self.client._odata._get_multiple.return_value = iter([]) + df = self.client.query.builder("account").top(10).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 0) + + +class TestExports(unittest.TestCase): + """col, raw, QueryResult are importable from models and package root.""" + + def test_col_importable_from_models(self): + from PowerPlatform.Dataverse.models import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_models(self): + from PowerPlatform.Dataverse.models import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_models(self): + from PowerPlatform.Dataverse.models import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_importable_from_package_root(self): + from PowerPlatform.Dataverse import col + + self.assertIsNotNone(col) + + def test_raw_importable_from_package_root(self): + from PowerPlatform.Dataverse import raw + + self.assertIsNotNone(raw) + + def test_query_result_importable_from_package_root(self): + from PowerPlatform.Dataverse import QueryResult + + self.assertIsNotNone(QueryResult) + + def test_col_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import col as root_col + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_col("statecode") == 0 + self.assertIsInstance(expr, FilterExpression) + + def test_raw_from_root_produces_filter_expression(self): + from PowerPlatform.Dataverse import raw as root_raw + from PowerPlatform.Dataverse.models.filters import FilterExpression + + expr = root_raw("statecode eq 0") + self.assertIsInstance(expr, FilterExpression) + + def test_query_result_from_root_is_correct_class(self): + from PowerPlatform.Dataverse import QueryResult as root_qr + + qr = root_qr([]) + self.assertIsInstance(qr, root_qr) + self.assertEqual(len(qr), 0) + + def test_col_from_package_root_no_warning(self): + """Importing col from package root fires no DeprecationWarning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import col # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + + def test_col_call_no_warning(self): + """col() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = col("statecode") == 0 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + def test_raw_call_no_warning(self): + """raw() emits no DeprecationWarning.""" + from PowerPlatform.Dataverse import raw + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = raw("statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"Unexpected warnings: {dep}") + self.assertIsNotNone(result) + + +class TestQueryResultAcceptanceCriteria(unittest.TestCase): + """Verify all QueryResult acceptance criteria from the spec.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + + def _execute(self): + return self.client.query.builder("account").select("name").execute() + + def test_result_is_query_result(self): + result = self._execute() + self.assertIsInstance(result, QueryResult) + + def test_for_loop_still_works(self): + result = self._execute() + records = [r for r in result] + self.assertEqual(len(records), 1) + + def test_first_returns_record_or_none(self): + result = self._execute() + r = result.first() + self.assertIsNotNone(r) + self.assertIsInstance(r, Record) + + def test_to_dataframe_returns_dataframe(self): + import pandas as pd + + result = self._execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + def test_builder_to_dataframe_returns_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "Contoso", "accountid": "id-1"}], + ] + ) + df = self.client.query.builder("account").select("name").to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase3_ga.py b/tests/unit/test_phase3_ga.py new file mode 100644 index 00000000..6802bc0a --- /dev/null +++ b/tests/unit/test_phase3_ga.py @@ -0,0 +1,404 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 3 GA regression tests. + +Covers: +- records.get() deprecation (DeprecationWarning, still functional) +- records.retrieve() — single record, None on 404 +- records.list() — QueryResult, accepts str/FilterExpression filter +- DataverseModel Protocol definition and isinstance() check +- DataverseModel exported from models and package root +- execute() emits zero DeprecationWarning (internal records.get() suppressed) + +Note: records.create(DataverseModel) and records.update(DataverseModel) dispatch +are deferred to post-GA and are not covered here. +""" + +import unittest +import warnings +from dataclasses import dataclass +from unittest.mock import MagicMock + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.core.errors import HttpError +from PowerPlatform.Dataverse.models.record import QueryResult, Record +from PowerPlatform.Dataverse.models.protocol import DataverseModel + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._get_multiple = MagicMock() + client._odata._get_single = MagicMock() + client._odata._get = MagicMock() + client._odata._create = MagicMock() + client._odata._create_multiple = MagicMock() + client._odata._update = MagicMock() + client._odata._update_by_ids = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + return client + + +# --------------------------------------------------------------------------- +# Sample DataverseModel implementation for tests + + +@dataclass +class _Account: + __entity_logical_name__ = "account" + __entity_set_name__ = "accounts" + name: str = "" + telephone1: str = "" + + def to_dict(self) -> dict: + return {"name": self.name, "telephone1": self.telephone1} + + @classmethod + def from_dict(cls, data: dict) -> "_Account": + return cls(name=data.get("name", ""), telephone1=data.get("telephone1", "")) + + +# --------------------------------------------------------------------------- + + +class TestDataverseModelProtocol(unittest.TestCase): + """DataverseModel Protocol structural checks.""" + + def test_account_satisfies_protocol(self): + self.assertIsInstance(_Account(), DataverseModel) + + def test_plain_dict_does_not_satisfy_protocol(self): + self.assertNotIsInstance({"name": "X"}, DataverseModel) + + def test_missing_entity_logical_name_fails(self): + class _Bad: + __entity_set_name__ = "bads" + + def to_dict(self): + return {} + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_missing_to_dict_fails(self): + class _Bad: + __entity_logical_name__ = "bad" + __entity_set_name__ = "bads" + + @classmethod + def from_dict(cls, d): + return cls() + + self.assertNotIsInstance(_Bad(), DataverseModel) + + def test_importable_from_models(self): + from PowerPlatform.Dataverse.models import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_importable_from_package_root(self): + from PowerPlatform.Dataverse import DataverseModel as dm + + self.assertIsNotNone(dm) + + def test_import_no_deprecation_warning(self): + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + from PowerPlatform.Dataverse import DataverseModel # noqa: F401 + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, []) + + +class TestRecordsGetDeprecation(unittest.TestCase): + """records.get() fires DeprecationWarning, still functional.""" + + def setUp(self): + self.client = _make_client() + + def test_get_single_warns(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with self.assertWarns(DeprecationWarning) as ctx: + self.client.records.get("account", "guid-1") + self.assertIn("retrieve", str(ctx.warning)) + + def test_get_multiple_warns(self): + self.client._odata._get_multiple.return_value = iter([]) + with self.assertWarns(DeprecationWarning) as ctx: + list(self.client.records.get("account", filter="statecode eq 0")) + self.assertIn("list", str(ctx.warning)) + + def test_get_single_still_returns_record(self): + self.client._odata._get.return_value = {"accountid": "1", "name": "Contoso"} + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + record = self.client.records.get("account", "guid-1") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_get_multiple_still_returns_pages(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A", "accountid": "1"}]]) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + pages = list(self.client.records.get("account", filter="statecode eq 0")) + self.assertEqual(len(pages), 1) + + def test_get_warning_message_single_id(self): + self.client._odata._get.return_value = {"accountid": "1"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.get("account", "guid-1") + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("retrieve" in m for m in msgs)) + + def test_get_warning_message_filter_form(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.get("account", filter="statecode eq 0")) + msgs = [str(w.message) for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertTrue(any("list" in m for m in msgs)) + + +class TestRecordsRetrieve(unittest.TestCase): + """records.retrieve() returns Record or None, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_retrieve_returns_record(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + record = self.client.records.retrieve("account", "abc") + self.assertIsInstance(record, Record) + self.assertEqual(record["name"], "Contoso") + + def test_retrieve_passes_select(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + self.client.records.retrieve("account", "abc", select=["name"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=None, include_annotations=None + ) + + def test_retrieve_passes_expand(self): + self.client._odata._get.return_value = { + "accountid": "abc", + "name": "Contoso", + "primarycontactid": {"contactid": "cid", "fullname": "John Doe"}, + } + record = self.client.records.retrieve("account", "abc", expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=["primarycontactid"], include_annotations=None + ) + self.assertEqual(record["primarycontactid"]["fullname"], "John Doe") + + def test_retrieve_passes_select_and_expand(self): + self.client._odata._get.return_value = { + "name": "Contoso", + "primarycontactid": {"fullname": "John Doe"}, + } + self.client.records.retrieve("account", "abc", select=["name"], expand=["primarycontactid"]) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=["name"], expand=["primarycontactid"], include_annotations=None + ) + + def test_retrieve_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get.return_value = { + "accountid": "abc", + "statuscode": 1, + f"statuscode@{annotation}": "Active", + } + record = self.client.records.retrieve("account", "abc", include_annotations=annotation) + self.client._odata._get.assert_called_once_with( + "account", "abc", select=None, expand=None, include_annotations=annotation + ) + self.assertEqual(record[f"statuscode@{annotation}"], "Active") + + def test_retrieve_no_deprecation_warning(self): + self.client._odata._get.return_value = {"accountid": "abc", "name": "Contoso"} + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.retrieve("account", "abc") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"retrieve() must not emit DeprecationWarning: {dep}") + + def test_retrieve_returns_none_on_404(self): + self.client._odata._get.side_effect = HttpError("Not Found", status_code=404) + result = self.client.records.retrieve("account", "nonexistent-guid") + self.assertIsNone(result) + + def test_retrieve_reraises_non_404(self): + self.client._odata._get.side_effect = HttpError("Server Error", status_code=500) + with self.assertRaises(HttpError): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_reraises_non_http_errors(self): + self.client._odata._get.side_effect = ValueError("Bad input") + with self.assertRaises(ValueError): + self.client.records.retrieve("account", "some-guid") + + def test_retrieve_record_id_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.id, "my-id") + + def test_retrieve_table_set(self): + self.client._odata._get.return_value = {"name": "Contoso"} + record = self.client.records.retrieve("account", "my-id") + self.assertEqual(record.table, "account") + + +class TestRecordsList(unittest.TestCase): + """records.list() returns QueryResult, no warning.""" + + def setUp(self): + self.client = _make_client() + + def test_list_returns_query_result(self): + self.client._odata._get_multiple.return_value = iter([]) + result = self.client.records.list("account") + self.assertIsInstance(result, QueryResult) + + def test_list_collects_all_pages(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}], + [{"name": "B", "accountid": "2"}, {"name": "C", "accountid": "3"}], + ] + ) + result = self.client.records.list("account") + self.assertEqual(len(result), 3) + + def test_list_no_deprecation_warning(self): + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.records.list("account", filter="statecode eq 0") + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"list() must not emit DeprecationWarning: {dep}") + + def test_list_passes_string_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", filter="statecode eq 0") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_filter_expression(self): + from PowerPlatform.Dataverse.models.filters import col + + self.client._odata._get_multiple.return_value = iter([]) + expr = col("statecode") == 0 + self.client.records.list("account", filter=expr) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["filter"], "statecode eq 0") + + def test_list_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", select=["name", "revenue"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["select"], ["name", "revenue"]) + + def test_list_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", top=50) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["top"], 50) + + def test_list_none_filter_passes_none(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account") + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertIsNone(call_kwargs["filter"]) + + def test_list_result_iterable(self): + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "X", "accountid": "1"}], + ] + ) + result = self.client.records.list("account") + records = list(result) + self.assertEqual(len(records), 1) + self.assertEqual(records[0]["name"], "X") + + def test_list_result_to_dataframe(self): + import pandas as pd + + self.client._odata._get_multiple.return_value = iter( + [ + [{"name": "A", "accountid": "1"}, {"name": "B", "accountid": "2"}], + ] + ) + df = self.client.records.list("account", select=["name"]).to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_list_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", orderby=["name asc"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["orderby"], ["name asc"]) + + def test_list_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", expand=["primarycontactid"]) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["expand"], ["primarycontactid"]) + + def test_list_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", page_size=200) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["page_size"], 200) + + def test_list_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", count=True) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertTrue(call_kwargs["count"]) + + def test_list_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + self.client.records.list("account", include_annotations=annotation) + call_kwargs = self.client._odata._get_multiple.call_args[1] + self.assertEqual(call_kwargs["include_annotations"], annotation) + + +class TestExecuteNoDeprecationFromRecordsGet(unittest.TestCase): + """execute() suppresses DeprecationWarning from the internal records.get() call.""" + + def setUp(self): + self.client = _make_client() + self.client._odata._get_multiple.return_value = iter([]) + + def test_execute_flat_no_warning(self): + from PowerPlatform.Dataverse.models.filters import col + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").where(col("statecode") == 0).execute() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(dep, [], f"execute() leaked DeprecationWarning: {dep}") + + def test_to_dataframe_no_records_get_warning(self): + """to_dataframe() emits its own deprecation but must not leak records.get()'s warning.""" + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.builder("account").select("name").to_dataframe() + dep = [w for w in caught if issubclass(w.category, DeprecationWarning)] + # Exactly one DeprecationWarning: from QueryBuilder.to_dataframe() itself. + # The internal records.get() deprecation must remain suppressed. + self.assertEqual(len(dep), 1, f"Unexpected warnings: {dep}") + self.assertIn("QueryBuilder.to_dataframe()", str(dep[0].message)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_phase4_ga.py b/tests/unit/test_phase4_ga.py new file mode 100644 index 00000000..76cdd90e --- /dev/null +++ b/tests/unit/test_phase4_ga.py @@ -0,0 +1,629 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +"""Phase 4 GA regression tests. + +Covers: +- fetchxml(): basic, pagination, missing-entity-element error +- sql_select / sql_join / sql_joins raise AttributeError (removed at GA) +- odata_select / odata_expand / odata_bind emit DeprecationWarning (deprecated at GA) +- sql_columns / odata_expands / sql() emit zero DeprecationWarning (still GA-clean) +""" + +import unittest +import warnings +import xml.etree.ElementTree as ET +from unittest.mock import MagicMock, patch + +from azure.core.credentials import TokenCredential + +from PowerPlatform.Dataverse.models.record import QueryResult + + +def _make_client(): + cred = MagicMock(spec=TokenCredential) + from PowerPlatform.Dataverse.client import DataverseClient + + client = DataverseClient("https://example.crm.dynamics.com", cred) + client._odata = MagicMock() + client._odata._entity_set_from_schema_name = MagicMock(side_effect=lambda t: t + "s") + client._odata.api = "https://example.crm.dynamics.com/api/data/v9.2" + return client + + +# --------------------------------------------------------------------------- +# fetchxml() +# --------------------------------------------------------------------------- + + +class TestFetchXml(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def _fetch_xml(self, entity="account"): + return f"""""" + + def _mock_response(self, records, more=False, cookie=""): + resp = MagicMock() + payload = {"value": records} + if more: + payload["@Microsoft.Dynamics.CRM.morerecords"] = True + payload["@Microsoft.Dynamics.CRM.fetchxmlpagingcookie"] = cookie + else: + payload["@Microsoft.Dynamics.CRM.morerecords"] = False + resp.json.return_value = payload + return resp + + def test_fetchxml_inert_no_http_request(self): + """fetchxml() alone must not fire any HTTP request.""" + from PowerPlatform.Dataverse.models.fetchxml_query import FetchXmlQuery + + query = self.client.query.fetchxml(self._fetch_xml()) + self.assertIsInstance(query, FetchXmlQuery) + self.client._odata._request.assert_not_called() + + def test_basic_returns_query_result(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "1"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + + def test_basic_record_count(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + + def test_record_values_accessible(self): + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso", "accountid": "abc-123"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(result.first()["name"], "Contoso") + + def test_empty_result_returns_empty_query_result(self): + self.client._odata._request.return_value = self._mock_response([]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertIsInstance(result, QueryResult) + self.assertEqual(len(result), 0) + self.assertFalse(result) + + def test_pagination_fetches_all_pages(self): + """execute_pages() drives the HTTP loop; each page yields one QueryResult.""" + # Annotation is outer XML; pagingcookie attribute is double URL-encoded inner cookie. + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_pagination_second_request_includes_page_and_cookie(self): + """execute_pages() injects the decoded paging cookie into the second request.""" + # pagingcookie="%253Cc%252F%253E": double URL-decode gives "" (the inner cookie XML). + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + second_call_kwargs = self.client._odata._request.call_args_list[1] + params = ( + second_call_kwargs.kwargs.get("params") or second_call_kwargs.args[2] + if len(second_call_kwargs.args) > 2 + else {} + ) + if not params: + params = second_call_kwargs[1].get("params", {}) + xml_sent = params.get("fetchXml", "") + fetch_el = ET.fromstring(xml_sent) + self.assertEqual(fetch_el.get("page"), "2") + self.assertIsNotNone(fetch_el.get("paging-cookie")) + + def test_missing_entity_element_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetchxml("") + self.assertIn("entity", str(ctx.exception).lower()) + + def test_entity_missing_name_attr_raises_value_error(self): + with self.assertRaises(ValueError) as ctx: + self.client.query.fetchxml("") + self.assertIn("name", str(ctx.exception).lower()) + + def test_entity_set_resolved_from_entity_name(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetchxml(self._fetch_xml("account")).execute() + self.client._odata._entity_set_from_schema_name.assert_called_with("account") + + def test_request_uses_prefer_header(self): + self.client._odata._request.return_value = self._mock_response([]) + self.client.query.fetchxml(self._fetch_xml()).execute() + call_kwargs = self.client._odata._request.call_args + headers = call_kwargs.kwargs.get("headers", {}) + self.assertIn("Prefer", headers) + self.assertIn("fetchxmlpagingcookie", headers["Prefer"]) + + def test_result_iterable(self): + self.client._odata._request.return_value = self._mock_response([{"name": "A"}, {"name": "B"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + names = [r["name"] for r in result] + self.assertEqual(names, ["A", "B"]) + + def test_result_to_dataframe(self): + try: + import pandas as pd + except ImportError: + self.skipTest("pandas not installed") + self.client._odata._request.return_value = self._mock_response([{"name": "Contoso"}, {"name": "Fabrikam"}]) + result = self.client.query.fetchxml(self._fetch_xml()).execute() + df = result.to_dataframe() + self.assertIsInstance(df, pd.DataFrame) + self.assertEqual(len(df), 2) + + def test_no_deprecation_warning_emitted(self): + self.client._odata._request.return_value = self._mock_response([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.fetchxml(self._fetch_xml()).execute() + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0, "fetchxml().execute() should not emit DeprecationWarning") + + def test_execute_pages_returns_iterator_of_query_result(self): + """execute_pages() yields QueryResult objects, one per HTTP page.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_execute_pages_one_http_call_per_page(self): + """Each execute_pages() iteration fires exactly one HTTP request.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + count = 0 + for _page in self.client.query.fetchxml(self._fetch_xml()).execute_pages(): + count += 1 + self.assertEqual(self.client._odata._request.call_count, 2) + self.assertEqual(count, 2) + + def test_execute_pages_per_page_records(self): + """Each page yielded by execute_pages() contains only its own records.""" + cookie_raw = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + page2 = self._mock_response([{"name": "B"}, {"name": "C"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + pages = list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + self.assertEqual(pages[0].first()["name"], "A") + self.assertEqual(pages[1].first()["name"], "B") + + # ------------------------------------------------------------------ + # Input validation + # ------------------------------------------------------------------ + + def test_non_string_input_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(123) + + def test_empty_string_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_whitespace_only_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml(" ") + + def test_malformed_xml_raises_validation_error(self): + from PowerPlatform.Dataverse.core.errors import ValidationError + + with self.assertRaises(ValidationError): + self.client.query.fetchxml("") + + def test_url_too_long_raises_validation_error(self): + """XML whose URL-encoded form exceeds 32,768 chars is rejected before any HTTP.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + # Alphanumeric chars are URL-safe and don't expand; a 32,769-char name attribute + # value pushes the encoded XML over the limit. + long_name = "a" * 32_769 + big_xml = f'' + with self.assertRaises(ValidationError): + self.client.query.fetchxml(big_xml) + + # ------------------------------------------------------------------ + # Paging behaviour + # ------------------------------------------------------------------ + + def test_morerecords_string_true_continues_paging(self): + """morerecords annotation as string "true" (not bool) is handled correctly.""" + cookie_raw = '' + page1_payload = { + "value": [{"name": "A"}], + "@Microsoft.Dynamics.CRM.morerecords": "true", + "@Microsoft.Dynamics.CRM.fetchxmlpagingcookie": cookie_raw, + } + page2_payload = { + "value": [{"name": "B"}], + "@Microsoft.Dynamics.CRM.morerecords": False, + } + r1, r2 = MagicMock(), MagicMock() + r1.json.return_value = page1_payload + r2.json.return_value = page2_payload + self.client._odata._request.side_effect = [r1, r2] + + result = self.client.query.fetchxml(self._fetch_xml()).execute() + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_simple_paging_fallback_emits_user_warning(self): + """No cookie returned with morerecords=True triggers a UserWarning.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("simple paging", str(user_warnings[0].message).lower()) + + def test_simple_paging_fallback_fetches_all_pages(self): + """Simple paging fallback continues iterating; caller still gets all records.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True): + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + self.assertEqual(self.client._odata._request.call_count, 2) + + def test_malformed_cookie_xml_warns_distinctly(self): + """A cookie that is not valid XML emits a 'could not be parsed' warning, not the no-cookie warning.""" + page1 = self._mock_response([{"name": "A"}], more=True, cookie="not-valid-xml") + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) + + def test_corrupt_pagenumber_warns_distinctly(self): + """Valid XML cookie with non-integer pagenumber emits a 'could not be parsed' warning.""" + bad_cookie = '' + page1 = self._mock_response([{"name": "A"}], more=True, cookie=bad_cookie) + page2 = self._mock_response([{"name": "B"}], more=False) + self.client._odata._request.side_effect = [page1, page2] + + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + result = self.client.query.fetchxml(self._fetch_xml()).execute() + + self.assertEqual(len(result), 2) + user_warnings = [w for w in caught if issubclass(w.category, UserWarning)] + self.assertEqual(len(user_warnings), 1) + self.assertIn("could not be parsed", str(user_warnings[0].message).lower()) + + def test_max_pages_exceeded_raises(self): + """Paging loop raises ValidationError after exceeding _MAX_PAGES.""" + from PowerPlatform.Dataverse.core.errors import ValidationError + + cookie_raw = '' + always_more = self._mock_response([{"name": "A"}], more=True, cookie=cookie_raw) + self.client._odata._request.return_value = always_more + + with patch("PowerPlatform.Dataverse.models.fetchxml_query._MAX_PAGES", 3): + with self.assertRaises(ValidationError): + list(self.client.query.fetchxml(self._fetch_xml()).execute_pages()) + + +# --------------------------------------------------------------------------- +# Removed SQL helpers — raise AttributeError +# --------------------------------------------------------------------------- + + +class TestRemovedSqlHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_select_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") + + def test_sql_joins_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") + + def test_sql_join_raises_attribute_error(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") + + +# --------------------------------------------------------------------------- +# Deprecated OData helpers — emit DeprecationWarning, still functional +# --------------------------------------------------------------------------- + + +class TestDeprecatedOdataHelpers(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + # --- odata_select --- + + def test_odata_select_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + + def test_odata_select_still_returns_list(self): + self.client._odata._list_columns.return_value = [ + { + "LogicalName": "name", + "AttributeType": "String", + "IsPrimaryId": False, + "IsPrimaryName": True, + "DisplayName": {}, + } + ] + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") + self.assertIsInstance(cols, list) + self.assertIn("name", cols) + + # --- odata_expand --- + + def _contact_to_account_rel(self): + return [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + } + ] + + def test_odata_expand_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + + def test_odata_expand_still_returns_nav_property(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + nav = self.client.query.odata_expand("contact", "account") + self.assertEqual(nav, "parentcustomerid_account") + + def test_odata_expand_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_expand("contact", "nonexistent") + + # --- odata_bind --- + + def test_odata_bind_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_odata_bind_still_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._contact_to_account_rel() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", "guid-123") + self.assertIsInstance(result, dict) + key = list(result.keys())[0] + self.assertEqual(key, "parentcustomerid_account@odata.bind") + self.assertIn("guid-123", result[key]) + + def test_odata_bind_no_match_raises_value_error(self): + self.client._odata._list_table_relationships.return_value = [] + with self.assertRaises(ValueError): + self.client.query.odata_bind("contact", "nonexistent", "guid") + + +# --------------------------------------------------------------------------- +# GA-clean methods: no DeprecationWarning +# --------------------------------------------------------------------------- + + +class TestGaCleanMethods(unittest.TestCase): + + def setUp(self): + self.client = _make_client() + + def test_sql_columns_no_warning(self): + self.client._odata._list_columns.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql_columns("account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_odata_expands_no_warning(self): + self.client._odata._list_table_relationships.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.odata_expands("contact") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_sql_no_warning(self): + self.client._odata._query_sql.return_value = [] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + self.client.query.sql("SELECT name FROM account") + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + def test_builder_no_warning(self): + self.client._odata._get_multiple.return_value = iter([[]]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.query.builder("account").select("name").execute()) + deps = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deps), 0) + + +# --------------------------------------------------------------------------- +# Codemod — execute(by_page=...) transforms +# --------------------------------------------------------------------------- + + +class TestCodemodByPage(unittest.TestCase): + """migrate_source() rewrites literal by_page arguments.""" + + @classmethod + def setUpClass(cls): + try: + from tools.migrate_v0_to_v1 import migrate_source + + cls.migrate = staticmethod(migrate_source) + except ImportError: + cls.migrate = None + + def setUp(self): + if self.migrate is None: + self.skipTest("libcst not installed or tools package not on path") + + def test_execute_by_page_true_becomes_execute_pages(self): + src = "result = builder.execute(by_page=True)\n" + out = self.migrate(src) + self.assertIn("execute_pages()", out) + self.assertNotIn("by_page", out) + self.assertNotIn("execute(", out) + + def test_execute_by_page_false_removes_flag(self): + src = "result = builder.execute(by_page=False)\n" + out = self.migrate(src) + self.assertIn("execute()", out) + self.assertNotIn("by_page", out) + + def test_execute_by_page_variable_not_rewritten(self): + """Variable by_page argument must not be rewritten — requires manual review.""" + src = "result = builder.execute(by_page=flag)\n" + out = self.migrate(src) + self.assertIn("by_page=flag", out) + + def test_idempotent_execute_pages(self): + """Codemod is idempotent — running again changes nothing.""" + src = "result = builder.execute(by_page=True)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + def test_idempotent_execute_no_flag(self): + src = "result = builder.execute(by_page=False)\n" + once = self.migrate(src) + twice = self.migrate(once) + self.assertEqual(once, twice) + + def test_client_var_default_rewrites_client(self): + """Default client_var='client' rewrites client.create(...).""" + src = "client.create('account', data)\n" + out = self.migrate(src) + self.assertIn("client.records.create", out) + + def test_client_var_custom_rewrites_matching_name(self): + """custom client_var rewrites that variable name, not 'client'.""" + src = "svc.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertIn("svc.records.create", out) + + def test_client_var_custom_does_not_rewrite_default_name(self): + """When client_var='svc', the literal name 'client' is left untouched.""" + src = "client.create('account', data)\n" + out = self.migrate(src, client_var="svc") + self.assertNotIn("client.records.create", out) + self.assertIn("client.create", out) + + +class TestManualReviewFinder(unittest.TestCase): + """find_manual_patterns() detects patterns that require manual migration.""" + + @classmethod + def setUpClass(cls): + try: + from tools.migrate_v0_to_v1 import find_manual_patterns + + cls.find = staticmethod(find_manual_patterns) + except ImportError: + cls.find = None + + def setUp(self): + if self.find is None: + self.skipTest("libcst not installed or tools package not on path") + + def test_records_get_flagged(self): + src = "result = client.records.get('account', record_id)\n" + findings = self.find(src) + self.assertTrue(any("records.get" in f for f in findings)) + + def test_dataframe_get_flagged(self): + src = "df = client.dataframe.get('account', select=['name'])\n" + findings = self.find(src) + self.assertTrue(any("dataframe.get" in f for f in findings)) + + def test_execute_by_page_variable_flagged(self): + src = "result = builder.execute(by_page=flag)\n" + findings = self.find(src) + self.assertTrue(any("by_page" in f for f in findings)) + + def test_execute_by_page_literal_not_flagged(self): + """Literal True/False is handled by the transformer — not a manual item.""" + src = "result = builder.execute(by_page=True)\n" + findings = self.find(src) + self.assertFalse(any("by_page" in f for f in findings)) + + def test_sql_select_flagged(self): + src = "cols = client.query.sql_select('account')\n" + findings = self.find(src) + self.assertTrue(any("sql_select" in f for f in findings)) + + def test_sql_join_flagged(self): + src = "j = client.query.sql_join('account', 'contact')\n" + findings = self.find(src) + self.assertTrue(any("sql_join" in f for f in findings)) + + def test_clean_code_no_findings(self): + src = "result = client.records.list('account', filter='statecode eq 0')\n" + self.assertEqual(self.find(src), []) + + def test_custom_client_var(self): + src = "svc.records.get('account', guid)\n" + self.assertEqual(self.find(src, client_var="client"), []) + findings = self.find(src, client_var="svc") + self.assertTrue(any("records.get" in f for f in findings)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/unit/test_query_operations.py b/tests/unit/test_query_operations.py index 1add6288..c6520e74 100644 --- a/tests/unit/test_query_operations.py +++ b/tests/unit/test_query_operations.py @@ -179,9 +179,13 @@ def test_builder_returns_query_builder(self): def test_builder_execute_flat_default(self): """builder().execute() should return flat records by default.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1", "name": "Test"}]]) - records = list(self.client.query.builder("account").select("name").filter_eq("statecode", 0).top(10).execute()) + records = list( + self.client.query.builder("account").select("name").where(col("statecode") == 0).top(10).execute() + ) self.client._odata._get_multiple.assert_called_once_with( "account", @@ -220,13 +224,15 @@ def test_builder_execute_by_page(self): def test_builder_execute_all_params(self): """builder().execute() should forward all parameters.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(50) @@ -248,13 +254,13 @@ def test_builder_execute_all_params(self): def test_builder_execute_with_where(self): """builder().where().execute() should compile expression to filter.""" - from PowerPlatform.Dataverse.models.filters import eq, gt + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"name": "Test"}]]) list( self.client.query.builder("account") - .where((eq("statecode", 0) | eq("statecode", 1)) & gt("revenue", 100000)) + .where(((col("statecode") == 0) | (col("statecode") == 1)) & (col("revenue") > 100000)) .execute() ) @@ -265,10 +271,12 @@ def test_builder_execute_with_where(self): ) def test_builder_execute_with_filter_in(self): - """builder().filter_in().execute() should forward CRM.In filter to _get_multiple.""" + """builder().where(col().in_()).execute() should forward CRM.In filter to _get_multiple.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_in("statecode", [0, 1, 2]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").in_([0, 1, 2])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -277,13 +285,15 @@ def test_builder_execute_with_filter_in(self): ) def test_builder_execute_with_where_filter_in(self): - """builder().where(filter_in(...) & ...).execute() should compile composed expression.""" - from PowerPlatform.Dataverse.models.filters import filter_in, gt + """builder().where(col().in_() & ...).execute() should compile composed expression.""" + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) list( - self.client.query.builder("account").where(filter_in("statecode", [0, 1]) & gt("revenue", 100000)).execute() + self.client.query.builder("account") + .where(col("statecode").in_([0, 1]) & (col("revenue") > 100000)) + .execute() ) call_kwargs = self.client._odata._get_multiple.call_args @@ -293,14 +303,15 @@ def test_builder_execute_with_where_filter_in(self): ) def test_builder_execute_with_filter_between_datetimes(self): - """builder().filter_between() with datetimes should forward correct OData.""" + """builder().where(col().between()).execute() should forward correct OData.""" from datetime import datetime, timezone + from PowerPlatform.Dataverse.models.filters import col self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) start = datetime(2024, 1, 1, 0, 0, 0, tzinfo=timezone.utc) end = datetime(2024, 12, 31, 23, 59, 59, tzinfo=timezone.utc) - list(self.client.query.builder("account").filter_between("createdon", start, end).execute()) + list(self.client.query.builder("account").where(col("createdon").between(start, end)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -309,10 +320,12 @@ def test_builder_execute_with_filter_between_datetimes(self): ) def test_builder_execute_with_filter_not_in(self): - """builder().filter_not_in().execute() should forward CRM.NotIn filter.""" + """builder().where(col().not_in()).execute() should forward CRM.NotIn filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").select("name").filter_not_in("statecode", [2, 3]).execute()) + list(self.client.query.builder("account").select("name").where(col("statecode").not_in([2, 3])).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -321,10 +334,12 @@ def test_builder_execute_with_filter_not_in(self): ) def test_builder_execute_with_filter_not_between(self): - """builder().filter_not_between().execute() should forward negated between filter.""" + """builder().where(col().not_between()).execute() should forward negated between filter.""" + from PowerPlatform.Dataverse.models.filters import col + self.client._odata._get_multiple.return_value = iter([[{"accountid": "1"}]]) - list(self.client.query.builder("account").filter_not_between("revenue", 100000, 500000).execute()) + list(self.client.query.builder("account").where(col("revenue").not_between(100000, 500000)).execute()) call_kwargs = self.client._odata._get_multiple.call_args self.assertEqual( @@ -334,6 +349,8 @@ def test_builder_execute_with_filter_not_between(self): def test_builder_full_fluent_workflow(self): """End-to-end test of the fluent query workflow.""" + from PowerPlatform.Dataverse.models.filters import col + expected_records = [ {"accountid": "1", "name": "Big Corp", "revenue": 5000000}, {"accountid": "2", "name": "Mega Inc", "revenue": 4000000}, @@ -343,8 +360,8 @@ def test_builder_full_fluent_workflow(self): records = list( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) - .filter_gt("revenue", 1000000) + .where(col("statecode") == 0) + .where(col("revenue") > 1000000) .order_by("revenue", descending=True) .expand("primarycontactid") .top(10) @@ -357,23 +374,24 @@ def test_builder_full_fluent_workflow(self): self.assertEqual(records[1]["name"], "Mega Inc") def test_builder_to_dataframe(self): - """builder().to_dataframe() should delegate to client.dataframe.get().""" + """builder().to_dataframe() should collect records into a DataFrame.""" import pandas as pd + from PowerPlatform.Dataverse.models.filters import raw - expected_df = pd.DataFrame([{"name": "Contoso", "revenue": 1000}]) - self.client.dataframe = MagicMock() - self.client.dataframe.get.return_value = expected_df + expected_records = [{"name": "Contoso", "revenue": 1000}] + self.client._odata._get_multiple.return_value = iter([expected_records]) result = ( self.client.query.builder("account") .select("name", "revenue") - .filter_eq("statecode", 0) + .where(raw("statecode eq 0")) .order_by("name") .top(50) + .execute() .to_dataframe() ) - self.client.dataframe.get.assert_called_once_with( + self.client._odata._get_multiple.assert_called_once_with( "account", select=["name", "revenue"], filter="statecode eq 0", @@ -384,7 +402,9 @@ def test_builder_to_dataframe(self): count=False, include_annotations=None, ) - pd.testing.assert_frame_equal(result, expected_df) + self.assertIsInstance(result, pd.DataFrame) + self.assertEqual(len(result), 1) + self.assertEqual(result.iloc[0]["name"], "Contoso") # =================================================================== @@ -526,239 +546,25 @@ def test_excludes_attribute_of_columns(self): self.assertNotIn("createdbyname", names) -class TestSqlSelect(unittest.TestCase): - """Tests for client.query.sql_select().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def test_returns_comma_separated(self): - self.client._odata._list_columns.return_value = [ - { - "LogicalName": "accountid", - "AttributeType": "Uniqueidentifier", - "IsPrimaryId": True, - "IsPrimaryName": False, - "DisplayName": {}, - }, - { - "LogicalName": "name", - "AttributeType": "String", - "IsPrimaryId": False, - "IsPrimaryName": True, - "DisplayName": {}, - }, - { - "LogicalName": "revenue", - "AttributeType": "Money", - "IsPrimaryId": False, - "IsPrimaryName": False, - "DisplayName": {}, - }, - ] - result = self.client.query.sql_select("account") - self.assertIn("accountid", result) - self.assertIn("name", result) - self.assertIn("revenue", result) - self.assertEqual(result.count(","), 2) # 3 cols = 2 commas - - -class TestSqlJoins(unittest.TestCase): - """Tests for client.query.sql_joins().""" - - def setUp(self): - self.mock_credential = MagicMock(spec=TokenCredential) - self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) - self.client._odata = MagicMock() - - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_outgoing_lookups(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 1) - j = joins[0] - self.assertEqual(j["column"], "parentcustomerid") - self.assertEqual(j["target"], "account") - self.assertEqual(j["target_pk"], "accountid") - self.assertIn("JOIN account", j["join_clause"]) - self.assertIn("parentcustomerid", j["join_clause"]) - - def test_ignores_incoming_rels(self): - self._mock_rels( - [ - # This is an incoming relationship (account is referenced, not referencing) - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - ] - ) - joins = self.client.query.sql_joins("account") - self.assertEqual(len(joins), 0) - - def test_polymorphic_returns_multiple(self): - self._mock_rels( - [ - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "opp_customer_accounts", - }, - { - "ReferencingEntity": "opportunity", - "ReferencingAttribute": "customerid", - "ReferencedEntity": "contact", - "ReferencedAttribute": "contactid", - "SchemaName": "opp_customer_contacts", - }, - ] - ) - joins = self.client.query.sql_joins("opportunity") - self.assertEqual(len(joins), 2) - targets = {j["target"] for j in joins} - self.assertEqual(targets, {"account", "contact"}) - # Both use the same source column - self.assertTrue(all(j["column"] == "customerid" for j in joins)) - - def test_empty_relationships(self): - self._mock_rels([]) - joins = self.client.query.sql_joins("account") - self.assertEqual(joins, []) - - def test_alias_collision_same_first_letter(self): - """Two targets starting with the same letter get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "regardingobjectid", - "ReferencedEntity": "annotation", - "ReferencedAttribute": "annotationid", - "SchemaName": "contact_annotation", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - def test_alias_collision_same_target_table(self): - """Two lookups to the same table (e.g. ownerid + createdby -> systemuser) get distinct aliases.""" - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_ownerid_systemuser", - }, - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "createdby", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_createdby_systemuser", - }, - ] - ) - joins = self.client.query.sql_joins("contact") - self.assertEqual(len(joins), 2) - aliases = [j["join_clause"].split()[2] for j in joins] - self.assertEqual(len(set(aliases)), 2, "aliases must be unique") - self.assertNotEqual(aliases[0], aliases[1]) - - -class TestSqlJoin(unittest.TestCase): - """Tests for client.query.sql_join().""" +class TestRemovedSqlHelpers(unittest.TestCase): + """sql_select(), sql_join(), sql_joins() are removed at GA — raise AttributeError.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def _mock_rels(self, rels): - self.client._odata._list_table_relationships.return_value = rels - - def test_generates_join_clause(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account", from_alias="c", to_alias="a") - self.assertEqual(result, "JOIN account a ON c.parentcustomerid = a.accountid") - - def test_default_aliases(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "SchemaName": "contact_customer_accounts", - }, - ] - ) - result = self.client.query.sql_join("contact", "account") - self.assertIn("JOIN account a ON contact.parentcustomerid = a.accountid", result) + def test_sql_select_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_select("account") - def test_no_relationship_raises(self): - self._mock_rels([]) - with self.assertRaises(ValueError) as ctx: - self.client.query.sql_join("contact", "nonexistent") - self.assertIn("No relationship found", str(ctx.exception)) + def test_sql_joins_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_joins("contact") - def test_case_insensitive_target(self): - self._mock_rels( - [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "ownerid", - "ReferencedEntity": "systemuser", - "ReferencedAttribute": "systemuserid", - "SchemaName": "contact_owner", - }, - ] - ) - result = self.client.query.sql_join("contact", "SystemUser", from_alias="c", to_alias="su") - self.assertIn("JOIN systemuser su", result) - self.assertIn("c.ownerid = su.systemuserid", result) + def test_sql_join_removed(self): + with self.assertRaises(AttributeError): + self.client.query.sql_join("contact", "account") # =================================================================== @@ -767,13 +573,18 @@ def test_case_insensitive_target(self): class TestOdataSelect(unittest.TestCase): - """Tests for client.query.odata_select().""" + """Tests for client.query.odata_select() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_columns.return_value = [] + with self.assertWarns(DeprecationWarning): + self.client.query.odata_select("account") + def test_returns_list_of_strings(self): self.client._odata._list_columns.return_value = [ { @@ -791,7 +602,11 @@ def test_returns_list_of_strings(self): "DisplayName": {}, }, ] - result = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_select("account") self.assertIsInstance(result, list) self.assertIn("accountid", result) self.assertIn("name", result) @@ -807,7 +622,11 @@ def test_result_usable_in_records_get(self): "DisplayName": {}, }, ] - cols = self.client.query.odata_select("account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + cols = self.client.query.odata_select("account") self.assertEqual(cols, ["name"]) @@ -914,13 +733,28 @@ def test_metadata_error_on_entity_set_resolution_is_swallowed(self): class TestOdataExpand(unittest.TestCase): - """Tests for client.query.odata_expand().""" + """Tests for client.query.odata_expand() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = [ + { + "ReferencingEntity": "contact", + "ReferencingAttribute": "parentcustomerid", + "ReferencedEntity": "account", + "ReferencedAttribute": "accountid", + "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", + "SchemaName": "contact_customer_accounts", + }, + ] + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_expand("contact", "account") + def test_returns_nav_property(self): self.client._odata._list_table_relationships.return_value = [ { @@ -933,7 +767,11 @@ def test_returns_nav_property(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "accounts" - result = self.client.query.odata_expand("contact", "account") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "account") self.assertEqual(result, "parentcustomerid_account") def test_no_relationship_raises(self): @@ -954,20 +792,24 @@ def test_case_insensitive_target(self): }, ] self.client._odata._entity_set_from_schema_name.return_value = "systemusers" - result = self.client.query.odata_expand("contact", "SystemUser") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_expand("contact", "SystemUser") self.assertEqual(result, "ownerid_systemuser") class TestOdataBind(unittest.TestCase): - """Tests for client.query.odata_bind().""" + """Tests for client.query.odata_bind() — deprecated at GA, still functional.""" def setUp(self): self.mock_credential = MagicMock(spec=TokenCredential) self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) self.client._odata = MagicMock() - def test_returns_bind_dict(self): - self.client._odata._list_table_relationships.return_value = [ + def _rel(self): + return [ { "ReferencingEntity": "contact", "ReferencingAttribute": "parentcustomerid", @@ -977,10 +819,23 @@ def test_returns_bind_dict(self): "SchemaName": "contact_customer_accounts", }, ] + + def test_emits_deprecation_warning(self): + self.client._odata._list_table_relationships.return_value = self._rel() + self.client._odata._entity_set_from_schema_name.return_value = "accounts" + with self.assertWarns(DeprecationWarning): + self.client.query.odata_bind("contact", "account", "some-guid") + + def test_returns_bind_dict(self): + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" guid = "12345678-1234-1234-1234-123456789abc" - result = self.client.query.odata_bind("contact", "account", guid) + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + result = self.client.query.odata_bind("contact", "account", guid) self.assertIsInstance(result, dict) self.assertEqual(len(result), 1) key = list(result.keys())[0] @@ -994,19 +849,14 @@ def test_no_relationship_raises(self): def test_usable_in_create_payload(self): """Result can be merged into a create payload via **spread.""" - self.client._odata._list_table_relationships.return_value = [ - { - "ReferencingEntity": "contact", - "ReferencingAttribute": "parentcustomerid", - "ReferencedEntity": "account", - "ReferencedAttribute": "accountid", - "ReferencingEntityNavigationPropertyName": "parentcustomerid_account", - "SchemaName": "contact_customer_accounts", - }, - ] + self.client._odata._list_table_relationships.return_value = self._rel() self.client._odata._entity_set_from_schema_name.return_value = "accounts" - bind = self.client.query.odata_bind("contact", "account", "some-guid") + import warnings + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + bind = self.client.query.odata_bind("contact", "account", "some-guid") payload = {"firstname": "Jane", "lastname": "Doe", **bind} self.assertIn("parentcustomerid_account@odata.bind", payload) self.assertEqual(payload["firstname"], "Jane") diff --git a/tests/unit/test_records_operations.py b/tests/unit/test_records_operations.py index 09df6869..b9ae98d8 100644 --- a/tests/unit/test_records_operations.py +++ b/tests/unit/test_records_operations.py @@ -389,5 +389,95 @@ def test_upsert_composite_alternate_key(self): self.client._odata._upsert_multiple.assert_not_called() +class TestListPages(unittest.TestCase): + """Unit tests for records.list_pages().""" + + def setUp(self): + self.mock_credential = MagicMock(spec=TokenCredential) + self.client = DataverseClient("https://example.crm.dynamics.com", self.mock_credential) + self.client._odata = MagicMock() + + def test_list_pages_returns_iterator(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + result = self.client.records.list_pages("account") + import types + + self.assertIsInstance(result, types.GeneratorType) + + def test_list_pages_yields_query_result_per_page(self): + from PowerPlatform.Dataverse.models.record import QueryResult + + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages), 2) + for page in pages: + self.assertIsInstance(page, QueryResult) + + def test_list_pages_page_contents(self): + self.client._odata._get_multiple.return_value = iter([[{"name": "A"}], [{"name": "B"}, {"name": "C"}]]) + pages = list(self.client.records.list_pages("account")) + self.assertEqual(len(pages[0]), 1) + self.assertEqual(len(pages[1]), 2) + + def test_list_pages_passes_filter(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", filter="statecode eq 0")) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("filter") or call_kwargs[1].get("filter"), "statecode eq 0") + + def test_list_pages_passes_select(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", select=["name"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("select") or call_kwargs[1].get("select"), ["name"]) + + def test_list_pages_passes_top(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", top=50)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("top") or call_kwargs[1].get("top"), 50) + + def test_list_pages_no_deprecation_warning(self): + import warnings + + self.client._odata._get_multiple.return_value = iter([]) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + list(self.client.records.list_pages("account", filter="statecode eq 0")) + deprecations = [w for w in caught if issubclass(w.category, DeprecationWarning)] + self.assertEqual(len(deprecations), 0) + + def test_list_pages_passes_orderby(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", orderby=["name asc"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("orderby"), ["name asc"]) + + def test_list_pages_passes_expand(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", expand=["primarycontactid"])) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("expand"), ["primarycontactid"]) + + def test_list_pages_passes_page_size(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", page_size=200)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("page_size"), 200) + + def test_list_pages_passes_count(self): + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", count=True)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertTrue(call_kwargs.kwargs.get("count")) + + def test_list_pages_passes_include_annotations(self): + annotation = "OData.Community.Display.V1.FormattedValue" + self.client._odata._get_multiple.return_value = iter([]) + list(self.client.records.list_pages("account", include_annotations=annotation)) + call_kwargs = self.client._odata._get_multiple.call_args + self.assertEqual(call_kwargs.kwargs.get("include_annotations"), annotation) + + if __name__ == "__main__": unittest.main() diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tools/migrate_v0_to_v1.py b/tools/migrate_v0_to_v1.py new file mode 100644 index 00000000..62678df8 --- /dev/null +++ b/tools/migrate_v0_to_v1.py @@ -0,0 +1,820 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT license. + +""" +DV-Python-SDK v0 -> v1 GA migration codemod. + +Mechanically rewrites beta (0.1.0b*) call sites to their GA (1.0) equivalents +using LibCST (concrete syntax tree — preserves all whitespace and comments). + +Usage:: + + pip install PowerPlatform-Dataverse-Client[migration] + dataverse-migrate path/to/your/scripts/ + dataverse-migrate path/to/your/scripts/ --dry-run # preview without writing + dataverse-migrate path/to/your/scripts/ --client-var=svc # if client is named 'svc' + + # Or via module for development installs: + python -m tools.migrate_v0_to_v1 path/to/your/scripts/ + +Transformations applied +----------------------- +Builder methods (.filter_* -> .where(col(...)...)):: + + .filter_eq("col", v) -> .where(col("col") == v) + .filter_ne("col", v) -> .where(col("col") != v) + .filter_gt("col", v) -> .where(col("col") > v) + .filter_ge("col", v) -> .where(col("col") >= v) + .filter_lt("col", v) -> .where(col("col") < v) + .filter_le("col", v) -> .where(col("col") <= v) + .filter_contains("col", v) -> .where(col("col").contains(v)) + .filter_startswith("col", v) -> .where(col("col").startswith(v)) + .filter_endswith("col", v) -> .where(col("col").endswith(v)) + .filter_in("col", vals) -> .where(col("col").in_(vals)) + .filter_not_in("col", vals) -> .where(col("col").not_in(vals)) + .filter_null("col") -> .where(col("col").is_null()) + .filter_not_null("col") -> .where(col("col").is_not_null()) + .filter_between("col", lo, hi) -> .where(col("col").between(lo, hi)) + .filter_not_between("col", lo, hi) -> .where(col("col").not_between(lo, hi)) + .filter_raw("expr") -> .where(raw("expr")) + .filter("expr") -> .where(raw("expr")) + .execute(by_page=True) -> .execute_pages() + .execute(by_page=False) -> .execute() (flag removed) + .to_dataframe() -> .execute().to_dataframe() + Inserts .execute() when the receiver is a recognised QueryBuilder chain + (contains .builder(), .select(), .where(), or a .filter_*() call). + +Record namespace:: + + batch.records.get(t, id) -> batch.records.retrieve(t, id) + +Top-level shortcuts (removed at GA):: + + client.create(t, d) -> client.records.create(t, d) + client.update(t, id, d) -> client.records.update(t, id, d) + client.delete(t, id) -> client.records.delete(t, id) + client.get(t, id) -> client.records.get(t, id) [deprecated; see manual section] + client.query_sql(sql) -> client.query.sql(sql) + client.get_table_info(t) -> client.tables.get(t) + client.create_table(t, …) -> client.tables.create(t, …) + client.delete_table(t) -> client.tables.delete(t) + client.list_tables() -> client.tables.list() + client.create_columns(t, …) -> client.tables.add_columns(t, …) + client.delete_columns(t, …) -> client.tables.remove_columns(t, …) + client.upload_file(…) -> client.files.upload(…) + +Import management: + Adds ``from PowerPlatform.Dataverse.models.filters import col`` when a + .filter_* method is rewritten (if col is not already imported). + Adds ``raw`` to the same import when .filter_raw or .filter is rewritten. + +NOT handled by this codemod (manual migration required): + execute(by_page=variable) -> manual review required (variable argument, not literal) + client.records.get(t, id) -> client.records.retrieve(t, id) + Return type changes: beta returns Record (raises on 404); GA retrieve() returns + Record | None. Callers that do not guard against None will fail silently. + client.records.get(t, kw=…) -> client.records.list(t, kw=…) + Return type changes: beta returns Iterable[List[Record]] (pages); GA list() + returns QueryResult (flat iterable over Records). Any ``for page in result: + for rec in page:`` iteration pattern breaks after a mechanical rename. + client.dataframe.get() -> client.query.builder(…).execute().to_dataframe() + Expression reconstruction requires understanding caller intent. + client.query.sql_select()/sql_join()/sql_joins() -> removed (no mechanical replacement) +""" + +from __future__ import annotations + +import sys +from pathlib import Path +from typing import List, Optional, Sequence, Set, Tuple + +try: + import libcst as cst +except ImportError as _e: + raise ImportError( + "libcst is required. Install with:\n" + " pip install PowerPlatform-Dataverse-Client[migration]\n" + " # or: pip install 'libcst>=1.0.0'" + ) from _e + + +# --------------------------------------------------------------------------- +# Filter-method -> .where(col(...)) mapping +# --------------------------------------------------------------------------- + +_UNARY_FILTER_MAP = { + "filter_null": "is_null", + "filter_not_null": "is_not_null", +} + +_BINARY_OP_MAP = { + "filter_eq": cst.Equal(), + "filter_ne": cst.NotEqual(), + "filter_gt": cst.GreaterThan(), + "filter_ge": cst.GreaterThanEqual(), + "filter_lt": cst.LessThan(), + "filter_le": cst.LessThanEqual(), +} + +_METHOD_FILTER_MAP = { + "filter_contains": "contains", + "filter_startswith": "startswith", + "filter_endswith": "endswith", + "filter_in": "in_", + "filter_not_in": "not_in", + "filter_between": "between", + "filter_not_between": "not_between", +} + +_ALL_FILTER_METHODS: Set[str] = set(_UNARY_FILTER_MAP) | set(_BINARY_OP_MAP) | set(_METHOD_FILTER_MAP) | {"filter_raw"} + +# Standalone filter functions from filters module (beta API) -> col() equivalents +# eq("f", v) -> col("f") == v, between("f", lo, hi) -> col("f").between(lo, hi), etc. +_FUNC_BINARY_OP_MAP = { + "eq": cst.Equal(), + "ne": cst.NotEqual(), + "gt": cst.GreaterThan(), + "ge": cst.GreaterThanEqual(), + "lt": cst.LessThan(), + "le": cst.LessThanEqual(), +} +_FUNC_METHOD_MAP = { + "contains": "contains", + "startswith": "startswith", + "endswith": "endswith", + "filter_in": "in_", + "not_in": "not_in", + "between": "between", + "not_between": "not_between", +} +_FUNC_UNARY_MAP = { + "is_null": "is_null", + "is_not_null": "is_not_null", +} +_ALL_FILTER_FUNCS: Set[str] = set(_FUNC_BINARY_OP_MAP) | set(_FUNC_METHOD_MAP) | set(_FUNC_UNARY_MAP) + +# Methods that identify a QueryBuilder call chain (used to detect .to_dataframe() callers) +_BUILDER_CHAIN_METHODS: Set[str] = {"builder", "select", "where", "filter", "execute_pages"} | _ALL_FILTER_METHODS + +# Top-level client shortcut -> (new_namespace, new_method) +_CLIENT_SHORTCUTS = { + "create": ("records", "create"), + "update": ("records", "update"), + "delete": ("records", "delete"), + "get": ("records", "get"), + "query_sql": ("query", "sql"), + "get_table_info": ("tables", "get"), + "create_table": ("tables", "create"), + "delete_table": ("tables", "delete"), + "list_tables": ("tables", "list"), + "create_columns": ("tables", "add_columns"), + "delete_columns": ("tables", "remove_columns"), + "upload_file": ("files", "upload"), +} + +_FILTERS_MODULE = "PowerPlatform.Dataverse.models.filters" + + +# --------------------------------------------------------------------------- +# Node helpers +# --------------------------------------------------------------------------- + + +def _name(s: str) -> cst.Name: + return cst.Name(s) + + +def _attr(obj: cst.BaseExpression, attr: str) -> cst.Attribute: + return cst.Attribute(value=obj, attr=cst.Name(attr)) + + +def _call(func: cst.BaseExpression, *args: cst.BaseExpression) -> cst.Call: + cst_args = [] + for i, a in enumerate(args): + comma = ( + cst.MaybeSentinel.DEFAULT if i == len(args) - 1 else cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")) + ) + cst_args.append(cst.Arg(value=a, comma=comma)) + return cst.Call(func=func, args=cst_args) + + +def _col_call(col_name_node: cst.BaseExpression) -> cst.Call: + """col("field_name") call node.""" + return _call(_name("col"), col_name_node) + + +def _filters_module_attr() -> cst.Attribute: + """Build the Attribute chain for PowerPlatform.Dataverse.models.filters.""" + return _attr( + _attr( + _attr(_name("PowerPlatform"), "Dataverse"), + "models", + ), + "filters", + ) + + +# --------------------------------------------------------------------------- +# Positional argument helpers +# --------------------------------------------------------------------------- + + +def _pos_arg(args: Sequence[cst.Arg], n: int) -> Optional[cst.BaseExpression]: + """Return the n-th (0-indexed) positional argument value, or None.""" + count = 0 + for a in args: + if a.keyword is None: + if count == n: + return a.value + count += 1 + return None + + +def _positional_count(args: Sequence[cst.Arg]) -> int: + return sum(1 for a in args if a.keyword is None) + + +# --------------------------------------------------------------------------- +# Main transformer +# --------------------------------------------------------------------------- + + +class _V1Migrator(cst.CSTTransformer): + """LibCST transformer rewriting DV-Python-SDK beta -> v1 GA.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self._needs_col = False + self._needs_raw = False + self._has_col = False + self._has_raw = False + # Names imported from filters module in this file (e.g. eq, gt, between) + self._imported_filter_funcs: Set[str] = set() + + # ------------------------------------------------------------------ + # Track existing col / raw imports + # ------------------------------------------------------------------ + + def visit_ImportFrom(self, node: cst.ImportFrom) -> None: + if isinstance(node.names, cst.ImportStar): + return + module_str = _dotted_name(node.module) + if module_str != _FILTERS_MODULE: + return + for alias in node.names: + name = alias.name.value if isinstance(alias.name, cst.Name) else "" + if name == "col": + self._has_col = True + elif name == "raw": + self._has_raw = True + elif name in _ALL_FILTER_FUNCS: + self._imported_filter_funcs.add(name) + + # ------------------------------------------------------------------ + # Rewrite call nodes + # ------------------------------------------------------------------ + + def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.BaseExpression: + func = updated_node.func + + # ---------------------------------------------------------------- + # Standalone filter functions: eq("f", v) -> col("f") == v, etc. + # Only transform names that were actually imported from filters module. + # Wrap Comparison nodes in explicit parentheses so that combining with + # & / | doesn't hit Python precedence bugs (& binds tighter than ==/>). + # ---------------------------------------------------------------- + if isinstance(func, cst.Name) and func.value in self._imported_filter_funcs: + result = self._build_filter_func_arg(func.value, updated_node.args) + if result is not None: + if isinstance(result, cst.Comparison): + result = result.with_changes(lpar=[cst.LeftParen()], rpar=[cst.RightParen()]) + return result + + if not isinstance(func, cst.Attribute): + return updated_node + + method_name = func.attr.value if isinstance(func.attr, cst.Name) else "" + + # ---------------------------------------------------------------- + # .filter_*(...) -> .where(col(...) ...) + # ---------------------------------------------------------------- + if method_name in _ALL_FILTER_METHODS: + where_arg = self._build_filter_arg(method_name, updated_node.args) + if where_arg is not None: + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=where_arg)], + ) + + # ---------------------------------------------------------------- + # .filter("expr") -> .where(raw("expr")) + # QueryBuilder.filter() was removed at GA (not deprecated). Wrapping + # in raw() preserves the OData string exactly for string-literal callers. + # ---------------------------------------------------------------- + if method_name == "filter": + expr_node = _pos_arg(updated_node.args, 0) + if expr_node is not None and _positional_count(updated_node.args) == 1: + self._needs_raw = True + return updated_node.with_changes( + func=func.with_changes(attr=_name("where")), + args=[cst.Arg(value=_call(_name("raw"), expr_node))], + ) + + # ---------------------------------------------------------------- + # .execute(by_page=True) -> .execute_pages() + # .execute(by_page=False) -> .execute() (flag removed) + # Only literal True/False are codemod-able; variable by_page requires + # manual review per section 8.5 of the GA spec. + # ---------------------------------------------------------------- + if method_name == "execute": + by_page_val = self._kwarg_bool_literal(updated_node.args, "by_page") + if by_page_val is True: + return updated_node.with_changes( + func=func.with_changes(attr=_name("execute_pages")), + args=[], + ) + if by_page_val is False: + other_args = [ + a + for a in updated_node.args + if not (isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page") + ] + return updated_node.with_changes(args=other_args) + + # ---------------------------------------------------------------- + # QueryBuilder.to_dataframe() -> .execute().to_dataframe() + # Only rewrites when the receiver is a recognised QueryBuilder chain + # (contains .builder(), .select(), .where(), or a .filter_*() call). + # Skips if receiver is already a .execute() call (QueryResult.to_dataframe() + # is the GA form and must not be touched). + # ---------------------------------------------------------------- + if method_name == "to_dataframe": + receiver = func.value + already_executed = ( + isinstance(receiver, cst.Call) + and isinstance(receiver.func, cst.Attribute) + and isinstance(receiver.func.attr, cst.Name) + and receiver.func.attr.value == "execute" + ) + if not already_executed and self._is_query_builder_chain(receiver): + execute_call = _call(_attr(receiver, "execute")) + return updated_node.with_changes(func=func.with_changes(value=execute_call)) + + # ---------------------------------------------------------------- + # batch.records.get(table, id) -> batch.records.retrieve(table, id) + # NOTE: client.records.get() is NOT codemodded — the return type changes + # between beta and GA (Record | None vs Record for single-id; QueryResult vs + # Iterable[List[Record]] for multi-record). Surrounding iteration patterns + # would silently break after a mechanical rename. + # ---------------------------------------------------------------- + if method_name == "get" and isinstance(func.value, cst.Attribute): + inner = func.value + if isinstance(inner.attr, cst.Name) and inner.attr.value == "records": + if isinstance(inner.value, cst.Name) and inner.value.value == "batch": + # batch.records.get() returns None in both versions — safe to rename + return updated_node.with_changes(func=func.with_changes(attr=_name("retrieve"))) + + # ---------------------------------------------------------------- + # client.(...) top-level shortcuts removed at GA + # Only match when receiver is the known client variable name to avoid + # false positives on record.get("field"), table_info.get("field"), etc. + # ---------------------------------------------------------------- + if ( + isinstance(func.value, cst.Name) + and func.value.value == self._client_var + and method_name in _CLIENT_SHORTCUTS + ): + new_ns, new_method = _CLIENT_SHORTCUTS[method_name] + new_func = _attr(_attr(func.value, new_ns), new_method) + return updated_node.with_changes(func=new_func) + + return updated_node + + # ------------------------------------------------------------------ + # Keyword-argument helpers + # ------------------------------------------------------------------ + + @staticmethod + def _kwarg_bool_literal(args: Sequence[cst.Arg], keyword: str) -> Optional[bool]: + """Return True/False if *keyword* is a literal bool kwarg, else None.""" + for a in args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == keyword: + if isinstance(a.value, cst.Name): + if a.value.value == "True": + return True + if a.value.value == "False": + return False + return None + + @staticmethod + def _is_query_builder_chain(node: cst.BaseExpression) -> bool: + """Return True if *node* is a call chain that includes a QueryBuilder method.""" + cur: cst.BaseExpression = node + while isinstance(cur, cst.Call): + f = cur.func + if isinstance(f, cst.Attribute) and isinstance(f.attr, cst.Name): + if f.attr.value in _BUILDER_CHAIN_METHODS: + return True + cur = f.value + else: + break + return False + + # ------------------------------------------------------------------ + # Build the argument for .where() from .filter_*() args + # ------------------------------------------------------------------ + + def _build_filter_arg( + self, + method_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + # .filter_raw(expr) -> raw(expr) + if method_name == "filter_raw": + self._needs_raw = True + return _call(_name("raw"), field_node) + + # .filter_null / .filter_not_null -> col("f").is_null() / .is_not_null() + if method_name in _UNARY_FILTER_MAP: + self._needs_col = True + proxy = _UNARY_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy)) + + # .filter_eq / .filter_ne / ... -> col("f") OP val + if method_name in _BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_BINARY_OP_MAP[method_name], + comparator=val_node, + ) + ], + ) + + # .filter_between / .filter_not_between -> col("f").between(lo, hi) + if method_name in ("filter_between", "filter_not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + # .filter_in / .filter_not_in / .filter_contains / etc. + if method_name in _METHOD_FILTER_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _METHOD_FILTER_MAP[method_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Standalone filter function: eq("f", v) -> col("f") == v, etc. + # ------------------------------------------------------------------ + + def _build_filter_func_arg( + self, + func_name: str, + args: Sequence[cst.Arg], + ) -> Optional[cst.BaseExpression]: + """Return the replacement expression node for a standalone filter call.""" + field_node = _pos_arg(args, 0) + if field_node is None: + return None + + if func_name in _FUNC_UNARY_MAP: + self._needs_col = True + proxy = _FUNC_UNARY_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy)) + + if func_name in _FUNC_BINARY_OP_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + return cst.Comparison( + left=_col_call(field_node), + comparisons=[ + cst.ComparisonTarget( + operator=_FUNC_BINARY_OP_MAP[func_name], + comparator=val_node, + ) + ], + ) + + if func_name in ("between", "not_between"): + lo = _pos_arg(args, 1) + hi = _pos_arg(args, 2) + if lo is None or hi is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), lo, hi) + + if func_name in _FUNC_METHOD_MAP: + val_node = _pos_arg(args, 1) + if val_node is None: + return None + self._needs_col = True + proxy = _FUNC_METHOD_MAP[func_name] + return _call(_attr(_col_call(field_node), proxy), val_node) + + return None + + # ------------------------------------------------------------------ + # Inject missing col / raw imports at module level + # ------------------------------------------------------------------ + + def leave_Module(self, original_node: cst.Module, updated_node: cst.Module) -> cst.Module: + to_add: List[str] = [] + if self._needs_col and not self._has_col: + to_add.append("col") + if self._needs_raw and not self._has_raw: + to_add.append("raw") + if not to_add: + return updated_node + + new_body = list(updated_node.body) + + # Try to augment an existing filters import line + for i, stmt in enumerate(new_body): + if not ( + isinstance(stmt, cst.SimpleStatementLine) + and len(stmt.body) == 1 + and isinstance(stmt.body[0], cst.ImportFrom) + ): + continue + imp = stmt.body[0] + if isinstance(imp.names, cst.ImportStar): + continue + if _dotted_name(imp.module) != _FILTERS_MODULE: + continue + existing_names = {alias.name.value for alias in imp.names if isinstance(alias.name, cst.Name)} + need = [n for n in to_add if n not in existing_names] + if not need: + return updated_node # already present + all_aliases = list(imp.names) + [cst.ImportAlias(name=_name(n)) for n in need] + # Re-apply commas + fixed = _comma_separated(all_aliases) + new_imp = imp.with_changes(names=fixed) + new_body[i] = stmt.with_changes(body=[new_imp]) + return updated_node.with_changes(body=new_body) + + # No existing filters import — insert a new one after the last import block + new_import_stmt = cst.SimpleStatementLine( + body=[ + cst.ImportFrom( + module=_filters_module_attr(), + names=_comma_separated([cst.ImportAlias(name=_name(n)) for n in to_add]), + ) + ] + ) + last_import_idx = 0 + for i, stmt in enumerate(new_body): + if isinstance(stmt, cst.SimpleStatementLine) and any( + isinstance(s, (cst.Import, cst.ImportFrom)) for s in stmt.body + ): + last_import_idx = i + new_body.insert(last_import_idx + 1, new_import_stmt) + return updated_node.with_changes(body=new_body) + + +def _comma_separated( + aliases: List[cst.ImportAlias], +) -> List[cst.ImportAlias]: + """Return aliases with commas between each, last one without.""" + result = [] + for i, alias in enumerate(aliases): + if i < len(aliases) - 1: + result.append(alias.with_changes(comma=cst.Comma(whitespace_after=cst.SimpleWhitespace(" ")))) + else: + result.append(alias.with_changes(comma=cst.MaybeSentinel.DEFAULT)) + return result + + +# --------------------------------------------------------------------------- +# Utility: dotted-name string from libcst Attribute / Name tree +# --------------------------------------------------------------------------- + + +def _dotted_name(node: Optional[cst.BaseExpression]) -> str: + if node is None: + return "" + if isinstance(node, cst.Name): + return node.value + if isinstance(node, cst.Attribute): + return f"{_dotted_name(node.value)}.{node.attr.value}" + return "" + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + +# --------------------------------------------------------------------------- +# Manual-review pattern detector +# --------------------------------------------------------------------------- + +_REMOVED_QUERY_METHODS: Set[str] = {"sql_select", "sql_join", "sql_joins"} + + +class _ManualReviewFinder(cst.CSTTransformer): + """Visitor that detects patterns the codemod cannot safely rewrite automatically.""" + + def __init__(self, client_var: str = "client") -> None: + self._client_var = client_var + self.findings: List[str] = [] + + def _receiver_chain(self, node: cst.Attribute) -> List[str]: + """Return the dotted name parts of an Attribute chain, innermost first.""" + parts: List[str] = [] + cur: cst.BaseExpression = node + while isinstance(cur, cst.Attribute): + if isinstance(cur.attr, cst.Name): + parts.append(cur.attr.value) + cur = cur.value + if isinstance(cur, cst.Name): + parts.append(cur.value) + return parts # e.g. ["get", "records", "client"] for client.records.get + + def visit_Call(self, node: cst.Call) -> None: + func = node.func + if not isinstance(func, cst.Attribute): + return + + method = func.attr.value if isinstance(func.attr, cst.Name) else "" + chain = self._receiver_chain(func) # [method, ns, client_var, ...] + + # execute(by_page=) — non-literal by_page cannot be codemodded + if method == "execute": + for a in node.args: + if isinstance(a.keyword, cst.Name) and a.keyword.value == "by_page": + if not (isinstance(a.value, cst.Name) and a.value.value in ("True", "False")): + self.findings.append( + "execute(by_page=) — non-literal by_page requires manual review; " + "replace with execute_pages() or execute() depending on runtime value" + ) + + # client.records.get() — return type changes make a mechanical rename unsafe + if method == "get" and len(chain) >= 3 and chain[1] == "records" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.records.get() — use retrieve() for single-record lookup " + "(return type changes: raises on 404 vs returns None) " + "or list() for multi-record (iteration pattern changes)" + ) + + # client.dataframe.get() — expression reconstruction requires understanding caller intent + if method == "get" and len(chain) >= 3 and chain[1] == "dataframe" and chain[2] == self._client_var: + self.findings.append( + f"{self._client_var}.dataframe.get() — use " + "query.builder(...).execute().to_dataframe(); requires manual reconstruction" + ) + + # client.query.sql_select/sql_join/sql_joins — removed with no mechanical replacement + if ( + method in _REMOVED_QUERY_METHODS + and len(chain) >= 3 + and chain[1] == "query" + and chain[2] == self._client_var + ): + self.findings.append(f"{self._client_var}.query.{method}() — removed at GA with no mechanical replacement") + + +def find_manual_patterns(source: str, *, client_var: str = "client") -> List[str]: + """Return descriptions of patterns in *source* that require manual migration.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError: + return [] + finder = _ManualReviewFinder(client_var=client_var) + tree.visit(finder) + return finder.findings + + +# --------------------------------------------------------------------------- +# File-level migration +# --------------------------------------------------------------------------- + + +def migrate_source(source: str, *, client_var: str = "client") -> str: + """Parse *source*, apply transformations, return migrated source.""" + try: + tree = cst.parse_module(source) + except cst.ParserSyntaxError as exc: + raise ValueError(f"Parse error: {exc}") from exc + new_tree = tree.visit(_V1Migrator(client_var=client_var)) + return new_tree.code + + +def migrate_file(path: Path, *, dry_run: bool = False, client_var: str = "client") -> Tuple[bool, List[str]]: + """Migrate *path* in place. Returns (was_changed, manual_review_notes).""" + original = path.read_text(encoding="utf-8") + try: + migrated = migrate_source(original, client_var=client_var) + except ValueError as exc: + print(f" [SKIP] {path}: {exc}", file=sys.stderr) + return False, [] + manual = find_manual_patterns(original, client_var=client_var) + changed = migrated != original + if changed and not dry_run: + path.write_text(migrated, encoding="utf-8") + return changed, manual + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def _collect_targets(paths: List[str]) -> List[Path]: + targets: List[Path] = [] + for p_str in paths: + p = Path(p_str) + if p.is_dir(): + root = p.resolve() + for candidate in sorted(p.rglob("*.py")): + resolved = candidate.resolve() + if root == resolved or root in resolved.parents: + targets.append(candidate) + else: + print(f"[WARN] Skipping symlink outside target directory: {candidate}", file=sys.stderr) + elif p.is_file() and p.suffix == ".py": + targets.append(p) + else: + print(f"[WARN] Not a file or directory: {p}", file=sys.stderr) + return targets + + +def main(argv: Optional[List[str]] = None) -> int: + args = sys.argv[1:] if argv is None else list(argv) + dry_run = "--dry-run" in args + client_var = "client" + remaining = [] + for a in args: + if a == "--dry-run": + continue + if a.startswith("--client-var="): + client_var = a[len("--client-var=") :] + else: + remaining.append(a) + + if not remaining: + print(__doc__) + print("\nUsage: dataverse-migrate [--dry-run] [--client-var=NAME] [ ...]") + return 1 + + targets = _collect_targets(remaining) + if not targets: + print("[ERROR] No Python files found.", file=sys.stderr) + return 1 + + changed = skipped = needs_manual = manual_total = 0 + for path in targets: + was_changed, notes = migrate_file(path, dry_run=dry_run, client_var=client_var) + if was_changed: + changed += 1 + tag = "[DRY-RUN]" if dry_run else "[MIGRATED]" + if notes: + print(f"{tag} {path} (auto-rewrites applied; manual review still required)") + else: + print(f"{tag} {path}") + elif notes: + needs_manual += 1 + print(f"[NEEDS-MANUAL] {path} (no auto-rewrites to apply; manual migration required)") + else: + skipped += 1 + for note in notes: + print(f" [MANUAL] {note}") + manual_total += 1 + + suffix = "would be " if dry_run else "" + parts = [f"{changed} file(s) {suffix}auto-migrated"] + if needs_manual: + parts.append(f"{needs_manual} need manual-only migration") + parts.append(f"{skipped} unchanged") + print(f"\nDone: {', '.join(parts)}.", end="") + if manual_total: + print(f" {manual_total} pattern(s) require manual review.") + else: + print() + return 0 + + +if __name__ == "__main__": + sys.exit(main())