Sky coverage (#18)

Sulla2012 · JBorrow · web-flow · commit f959e0985a74 · 2026-02-20T21:21:09.000-05:00
* feat: initial function for finding tiles from bounding box and tmap

* feat: update docs and make composite ID

* feat: init function to get a list of sky coverage table entries from a d1map

* fix: ruff formating

* fix: more ruff formating

* feat: switching from passing box to getting box using pixell

* feat: moving sky coverage tools

* fix: fixing composite key generation

* feat: starting skycoverage update function

* feat: add sqlite to gitignore

* fix: updating outerjoin query

* feat: move core functionaliy of ingestact to a separate function for testability

* feat: move core functionality of update_sky_coverage to its own function for testability

* feat: adding test for ingestact and update_sky_coverage

* fix: map=d1table is redundant

* fix: move update_sky_coverage

* fix: ruff

* Fix tests

* Add h5py as dependency

* Satisfy tyg

* fix: remove unneccesary update statement

* fix: remove unused imports

* feat: switch to env variables for file paths

* fix: updating to latest ruff

* temp: checking DEPTH_ONE_MAP path

* temp: echo DEPTH_ONE_PATH

* fix: wrong variable name for DEPTH_ONE_PARENT

---------

Co-authored-by: Josh Borrow &lt;josh@joshborrow.com&gt;
diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml
@@ -2,6 +2,9 @@ name: Run tests
 
 on: [push]
 
+env:
+  DEPTH_ONE_PARENT: "../.pytest_cache/d1maps/"
+
 jobs:
   runtest:
     env:
@@ -29,4 +32,4 @@ jobs:
 
       - name: Run tests
         run: pytest --cov
-        
+        
diff --git a/.gitignore b/.gitignore
@@ -1,5 +1,6 @@
 .venv/*
 *.db
+*.sqlite
 *__pycache__*
 dist/*
 *.egg-info
diff --git a/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py b/mapcat/alembic/versions/1195d17201ba_add_notes_missing_field_to_atomic_maps.py
@@ -6,16 +6,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "1195d17201ba"
-down_revision: Union[str, None] = "bc2be980cfe7"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "bc2be980cfe7"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
diff --git a/mapcat/alembic/versions/57b97937e1bc_.py b/mapcat/alembic/versions/57b97937e1bc_.py
@@ -6,16 +6,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "57b97937e1bc"
-down_revision: Union[str, None] = "78293df04081"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "78293df04081"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
diff --git a/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py b/mapcat/alembic/versions/6ce7e94dfd2d_add_filtered_masps.py
@@ -6,16 +6,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "6ce7e94dfd2d"
-down_revision: Union[str, None] = "976527c198bd"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "976527c198bd"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 NEW_COLUMNS_DEPTH_ONE = [
     "rho_path",
@@ -51,8 +51,6 @@ def upgrade() -> None:
             "depth_one_coadds", sa.Column(column_name, sa.String(), nullable=True)
         )
 
-    pass
-
 
 def downgrade() -> None:
     # Depth 1
@@ -63,5 +61,3 @@ def downgrade() -> None:
 
     for column_name in NEW_COLUMNS_DEPTH_ONE_COADDS:
         op.drop_column("depth_one_coadds", column_name)
-
-    pass
diff --git a/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py b/mapcat/alembic/versions/78293df04081_create_depth_1_map_table.py
@@ -7,16 +7,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "78293df04081"
-down_revision: Union[str, None] = None
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = None
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
@@ -139,14 +139,14 @@ def upgrade() -> None:
 
     op.create_table(
         "depth_one_sky_coverage",
-        sa.Column("patch_id", sa.Integer, primary_key=True),
-        sa.Column("x", sa.CHAR, nullable=False),
-        sa.Column("y", sa.CHAR, nullable=False),
+        sa.Column("x", sa.CHAR, nullable=False, primary_key=True),
+        sa.Column("y", sa.CHAR, nullable=False, primary_key=True),
         sa.Column(
             "map_id",
             sa.Integer,
             sa.ForeignKey("depth_one_maps.map_id", ondelete="CASCADE"),
             nullable=False,
+            primary_key=True,
         ),
     )
 
diff --git a/mapcat/alembic/versions/976527c198bd_add_indices.py b/mapcat/alembic/versions/976527c198bd_add_indices.py
@@ -6,15 +6,15 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "976527c198bd"
-down_revision: Union[str, None] = "fd6670a1fdbe"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "fd6670a1fdbe"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
diff --git a/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py b/mapcat/alembic/versions/bc2be980cfe7_add_notes_json_field_to_depth_one_maps.py
@@ -6,16 +6,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "bc2be980cfe7"
-down_revision: Union[str, None] = "57b97937e1bc"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "57b97937e1bc"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
diff --git a/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py b/mapcat/alembic/versions/fd6670a1fdbe_update_atomic_map_coadd_table.py
@@ -6,16 +6,16 @@
 
 """
 
-from typing import Sequence, Union
+from collections.abc import Sequence
 
 import sqlalchemy as sa
 from alembic import op
 
 # revision identifiers, used by Alembic.
 revision: str = "fd6670a1fdbe"
-down_revision: Union[str, None] = "1195d17201ba"
-branch_labels: Union[str, Sequence[str], None] = None
-depends_on: Union[str, Sequence[str], None] = None
+down_revision: str | None = "1195d17201ba"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
 
 
 def upgrade() -> None:
diff --git a/mapcat/database/__init__.py b/mapcat/database/__init__.py
@@ -13,14 +13,14 @@
 from .tod import TODDepthOneTable
 
 __all__ = [
-    "AtomicMapTable",
     "AtomicMapCoaddTable",
+    "AtomicMapTable",
     "DepthOneMapTable",
-    "TimeDomainProcessingTable",
-    "PointingResidualTable",
-    "TODDepthOneTable",
     "PipelineInformationTable",
+    "PointingResidualTable",
     "SkyCoverageTable",
+    "TODDepthOneTable",
+    "TimeDomainProcessingTable",
 ]
 
 ALL_TABLES = [
diff --git a/mapcat/database/depth_one_map.py b/mapcat/database/depth_one_map.py
@@ -66,6 +66,7 @@ class DepthOneMapTable(SQLModel, table=True):
         List of pipeline info associed with d1 map
     depth_one_sky_coverage : list[SkyCoverageTable]
         List of sky coverage patches for d1 map.
+
     notes: dict[str, Any]
         JSON entry that holds additional information about the d1 maps
     """
diff --git a/mapcat/database/pipeline_information.py b/mapcat/database/pipeline_information.py
@@ -36,5 +36,3 @@ class PipelineInformationTable(SQLModel, table=True):
     sotodlib_version: str
     map_maker: str
     preprocess_info: dict[str, Any] = Field(sa_type=JSON)
-
-    map: DepthOneMapTable = Relationship(back_populates="pipeline_information")
diff --git a/mapcat/database/sky_coverage.py b/mapcat/database/sky_coverage.py
@@ -2,34 +2,43 @@
 Sky coverage table.
 """
 
+from sqlalchemy import PrimaryKeyConstraint
 from sqlmodel import Field, Relationship, SQLModel
 
 from .depth_one_map import DepthOneMapTable
 
 
 class SkyCoverageTable(SQLModel, table=True):
     """
-    Table for tracking sky coverage for a depth one map. x and y are 0->36 and
-    0-18 respectively for CAR patches with 10x10 degrees each.
+    Table for tracking sky coverage patches with non-zero overlap with a given depth one map.
+    x and y are 0->36 and 0-18 respectively for CAR patches with 10x10 degrees each.
 
     Attributes
     ----------
-    id : str
-        Internal ID of the sky coverage
-    map_name : str
-        Name of depth 1 map being tracked. Foreign into DepthOneMap
-    patch_coverage : str
-        String which represents the sky coverage of the d1map
+    sky_cov_id : PrimaryKeyConstraint
+        Composite ID from map_id, x, and y
+    map : DepthOneMapTable
+       Depth 1 map being tracked. Foreign into DepthOneMap
+    map_id : int
+       ID of depth 1 map being tracked
+    x : int
+        x-index of coverage patch. x=0 runs from RA 0 to 10,, etc.
+    y : in
+        y-index of coverage patch. y=0 runs from dec = -90 to -80, etc.
     """
 
     __tablename__ = "depth_one_sky_coverage"
 
-    patch_id: int = Field(primary_key=True)
-
-    x: int = Field(index=True)
-    y: int = Field(index=True)
+    x: int = Field(index=True, primary_key=True)
+    y: int = Field(index=True, primary_key=True)
 
     map_id: int = Field(
-        foreign_key="depth_one_maps.map_id", nullable=False, ondelete="CASCADE"
+        foreign_key="depth_one_maps.map_id",
+        nullable=False,
+        ondelete="CASCADE",
+        primary_key=True,
     )
+
     map: DepthOneMapTable = Relationship(back_populates="depth_one_sky_coverage")
+
+    __table_args__ = (PrimaryKeyConstraint("map_id", "x", "y", name="sky_cov_id"),)
diff --git a/mapcat/helper.py b/mapcat/helper.py
@@ -2,6 +2,7 @@
 Helper for mapcat database.
 """
 
+import os
 from pathlib import Path
 from typing import Literal
 
@@ -16,10 +17,10 @@ class Settings(BaseSettings):  # pragma: no cover
     database_name: str = "mapcat.db"
     database_type: Literal["sqlite", "postgresql"] = "sqlite"
 
-    depth_one_coadd_parent: Path = "./"
-    depth_one_parent: Path = "./"
-    atomic_coadd_parent: Path = "./"
-    atomic_parent: Path = "./"
+    depth_one_coadd_parent: Path = Path(os.environ.get("DEPTH_ONE_COADD_PARENT", "./"))
+    depth_one_parent: Path = Path(os.environ.get("DEPTH_ONE_PARENT", "./"))
+    atomic_coadd_parent: Path = Path(os.environ.get("ATOMIC_COADD_PARENT", "./"))
+    atomic_parent: Path = Path(os.environ.get("ATOMIC_PARENT", "./"))
 
     echo: bool = False
 
@@ -74,5 +75,3 @@ def migrate():
     location = __file__.replace("helper.py", "alembic.ini")
 
     subprocess.call(["alembic", "-c", location, "upgrade", "head"])
-
-    return
diff --git a/mapcat/toolkit/act.py b/mapcat/toolkit/act.py
@@ -2,15 +2,14 @@
 Create objects from an ACT depth 1 map file.
 """
 
+import argparse as ap
 from pathlib import Path
 
 import h5py
+from sqlalchemy.orm import sessionmaker
 
 from mapcat.database import DepthOneMapTable, TODDepthOneTable
 
-# from astropy import units as u
-# from astropy.coordinates import SkyCoord
-
 
 def extract_string(input: bytes) -> str:
     return str(input).replace("b'", "").replace("'", "")
@@ -75,7 +74,7 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT
         for obs_id in file_info["observations"]
     ]
 
-    depth_one_map = DepthOneMapTable(
+    return DepthOneMapTable(
         map_name=filenames["map"].replace("_map.fits", ""),
         map_path=filenames["map"],
         ivar_path=filenames.get("ivar"),
@@ -88,8 +87,6 @@ def create_objects(base: str, relative_to: Path, telescope: str) -> DepthOneMapT
         tods=tods,
     )
 
-    return depth_one_map
-
 
 def glob(input_glob: str, relative_to: Path, telescope: str) -> list[DepthOneMapTable]:
     maps = []
@@ -130,9 +127,28 @@ def glob(input_glob: str, relative_to: Path, telescope: str) -> list[DepthOneMap
 """
 
 
-def main():
-    import argparse as ap
+def core(session: sessionmaker, args: ap.Namespace):
+    """
+    Driver function for act.py Takes a session and a arg parser
+    and creates DepthOneMapTable objects from the files listed
+    matching the glob patter in the parser and adds them to the
+    database in session.
+
+    Parameters
+    ----------
+    session : sessionmaker
+        A SQLAlchemy sessionmaker to use for database access.
+    args : argparse.Namespace
+       Parsed args with the glob patterns to match.
+    """
 
+    with session() as cur_session:
+        maps = glob(args.glob, args.relative_to, args.telescope)
+        cur_session.add_all(maps)
+        cur_session.commit()
+
+
+def main():
     from mapcat.helper import settings
 
     parser = ap.ArgumentParser(prog="actingest", usage=USAGE, description=HELP_TEXT)
@@ -163,7 +179,4 @@ def main():
 
     args = parser.parse_args()
 
-    with settings.session() as session:
-        maps = glob(args.glob, args.relative_to, args.telescope)
-        session.add_all(maps)
-        session.commit()
+    core(session=settings.session, args=args)
diff --git a/mapcat/toolkit/mapmaking.py b/mapcat/toolkit/mapmaking.py
diff --git a/mapcat/toolkit/update_sky_coverage.py b/mapcat/toolkit/update_sky_coverage.py
diff --git a/pyproject.toml b/pyproject.toml
diff --git a/tests/test_act.py b/tests/test_act.py
diff --git a/tests/test_mapcat.py b/tests/test_mapcat.py
diff --git a/tests/test_mapmaking.py b/tests/test_mapmaking.py
