refactor: add additional logging

also moved READER_CONFIG out of the try except block as it's setup error of the DVE rather than user/submitter error.

Author: georgeRobertson

src/dve/pipeline/pipeline.py

@@ -241,6 +241,7 @@ def write_file_to_parquet(
                         f"{out}{model_name}",
                     )
             except MessageBearingError as exc:
+                self._logger.error(f"Unable to process {model_name}", exc_info=exc)
                 errors.extend(exc.messages)
 
         return list(dict.fromkeys(errors))  # remove any duplicate errors

src/dve/pipeline/utils.py

@@ -56,21 +56,17 @@ def load_reader(
     """Loads the readers for the diven feed, model name and file extension"""
     try:
         reader_config = dataset[model_name].reader_config[f".{file_extension.lower()}"]
-        reader = _READER_REGISTRY[reader_config.reader](
-            **reader_config.kwargs_, **backend_reader_kwargs if backend_reader_kwargs else {}
-        )
-        return reader
     except KeyError as exc:
         if file_extension:
             err_msg = (
-                f"The supplied file extension `{file_extension if file_extension else None}`"
+                f"The supplied file extension `{file_extension}`"
                 +f" is not a supported file format for {model_name}."
             )
         else:
             err_msg = "No supplied file extension. Unable to parse file without a file extension."
 
         raise MessageBearingError(
-            "The file extension provided is not supported.",
+            f"The file extension provided ({file_extension}) is not supported for this collection.",
             messages=[
                 FeedbackMessage(
                     entity=model_name,
@@ -83,6 +79,11 @@ def load_reader(
             ],
         ) from exc
 
+    reader = _READER_REGISTRY[reader_config.reader](
+        **reader_config.kwargs_, **backend_reader_kwargs if backend_reader_kwargs else {}
+    )
+    return reader
+
 
 def unpersist_all_rdds(spark: SparkSession):
     """Unpersist any checkpointed or cached rdds to avoid memory leaks"""