build: bump pyspark to 3.5.5 and address review comments

stevenhsd · stevenhsd · commit d676871ced3d · 2026-07-13T23:26:00.000+01:00
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -43,7 +43,7 @@ pandas = "2.3.3"
 polars = "0.20.31"
 pyarrow = "23.0.1"
 pydantic = "1.10.19"
-pyspark = ">=3.5.0,<=3.5.2"
+pyspark = ">=3.5.0,<=3.5.5"
 typing_extensions = "4.15.0"
 urllib3 = "2.7.0"  # dependency of boto3 & botocore
 
diff --git a/src/dve/core_engine/backends/implementations/spark/contract.py b/src/dve/core_engine/backends/implementations/spark/contract.py
@@ -147,6 +147,7 @@ def apply_data_contract(
                 self.logger.info(f"Data contract found {msg_count} issues in {entity_name}")
 
             try:
+                # TODO: will need to revisit in pydantic v2 bump as model field no longer available
                 record_df = record_df.select(
                     *[
                         (

Original file line number	Diff line number	Diff line change
`@@ -147,6 +147,7 @@ def apply_data_contract(`
`147`	`147`	`self.logger.info(f"Data contract found {msg_count} issues in {entity_name}")`
`148`	`148`
`149`	`149`	`try:`
	`150`	`+ # TODO: will need to revisit in pydantic v2 bump as model field no longer available`
`150`	`151`	`record_df = record_df.select(`
`151`	`152`	`*[`
`152`	`153`	`(`