refactor: untrack generated python proto, regenerate in build

.github/workflows/build.yml

@@ -355,11 +355,21 @@ jobs:
         # mirrors a subset of common deps (e.g. pillow); without this
         # flag a dependabot bump to a version not yet mirrored there
         # fails to resolve even though PyPI has it.
+        # dev-requirements.txt provides the betterproto plugin used by
+        # bin/python-proto-gen.sh.
         run: |
           python -m pip install uv
           if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi
           if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi
           if [ -f amber/dev-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/dev-requirements.txt; fi
+      - name: Install protoc
+        # Version pinned in bin/protoc-version.txt.
+        run: |
+          PROTOC_VERSION=$(cat bin/protoc-version.txt)
+          curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+          sudo unzip -o /tmp/protoc.zip -d /usr/local
+          sudo chmod +x /usr/local/bin/protoc
+          sudo chmod -R a+rX /usr/local/include/google
       - name: Create Databases
         run: |
           psql -h localhost -U postgres -f sql/texera_ddl.sql
@@ -467,6 +477,10 @@ jobs:
             }
           }
           EOF
+      - name: Generate Python proto bindings
+        # Integration specs spawn Python UDF workers that import the
+        # generated betterproto bindings. Independent of sbt and the JDK.
+        run: bash bin/python-proto-gen.sh
       - name: Lint and run amber integration tests
         # AMBER_TEST_FILTER=integration-only tells amber/build.sbt to
         # keep only @org.apache.texera.amber.tags.IntegrationTest
@@ -685,6 +699,16 @@ jobs:
         run: |
           python -m pip install uv
           if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi
+      - name: Install protoc
+        # Version pinned in bin/protoc-version.txt.
+        run: |
+          PROTOC_VERSION=$(cat bin/protoc-version.txt)
+          curl -fsSL -o /tmp/protoc.zip "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip"
+          sudo unzip -o /tmp/protoc.zip -d /usr/local
+          sudo chmod +x /usr/local/bin/protoc
+          sudo chmod -R a+rX /usr/local/include/google
+      - name: Generate Python proto bindings
+        run: bash bin/python-proto-gen.sh
       - name: Test with pytest
         # --junit-xml emits a JUnit-XML report alongside the coverage XML
         # so the Test Analytics upload below can feed Codecov's failing-
@@ -745,6 +769,12 @@ jobs:
           if [ -f amber/requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/requirements.txt; fi
           if [ -f amber/operator-requirements.txt ]; then uv pip install --system --index-strategy unsafe-best-match -r amber/operator-requirements.txt; fi
           if [ -f amber/dev-requirements.txt ]; then uv pip install --system -r amber/dev-requirements.txt; fi
+      - name: Install protoc
+        # Homebrew protoc; this job doesn't exercise scalapb so the
+        # bin/protoc-version.txt pin doesn't apply here.
+        run: brew install protobuf
+      - name: Generate Python proto bindings
+        run: bash bin/python-proto-gen.sh
       - name: Run state-materialization integration tests
         run: |
           cd amber && pytest -sv \

.gitignore

@@ -60,6 +60,9 @@ coverage.xml
 *.model
 *.pkl
 
+# Regenerated by bin/python-proto-gen.sh.
+amber/src/main/python/proto/
+
 # Ingoring user generated resources
 user-resources/
 

AGENTS.md

@@ -83,6 +83,8 @@ One Python venv shared across worktrees, sibling of the texera checkout:
 ```bash
 python3.12 -m venv ../venv312 && source ../venv312/bin/activate
 pip install -r amber/requirements.txt -r amber/operator-requirements.txt
+# For pytest or running bin/python-proto-gen.sh, also install dev deps:
+pip install -r amber/dev-requirements.txt
 ```
 
 Tests that spawn Python workers need an interpreter path. Edit `python.path`

amber/build.sbt

@@ -179,7 +179,7 @@ libraryDependencies ++= hadoopDependencies
 // protobuf related
 // run the following with sbt to have protobuf codegen
 
-PB.protocVersion := "3.19.4"
+PB.protocVersion := IO.read((ThisBuild / baseDirectory).value / "bin" / "protoc-version.txt").trim
 
 enablePlugins(Fs2Grpc)
 

amber/dev-requirements.txt

@@ -23,3 +23,7 @@
 # Coverage instrumentation for pytest; emits coverage.xml consumed by
 # Codecov's Phase 1 upload.
 pytest-cov==5.0.0
+
+# protoc plugin for bin/python-proto-gen.sh; runtime needs only the
+# base `betterproto` in requirements.txt.
+betterproto[compiler]==2.0.0b7