[TBD] - Implement Docker support with image creation and configuration updates

.dockerignore

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*
+!Dockerfile
+!python/
+python/**
+!python/Dockerfile
+!python/docker/
+!python/docker/**
+!python/examples/
+!python/examples/**
+!python/pyproject.toml
+!python/setup.cfg
+!python/README.md
+!python/src/
+python/src/**
+!python/src/pywy/
+!python/src/pywy/**
+!wayang-assembly/
+wayang-assembly/**
+!wayang-assembly/target/
+wayang-assembly/target/**
+!wayang-assembly/target/*-dist.tar.gz

.github/workflows/docker.yml

@@ -0,0 +1,76 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+name: Docker
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+concurrency:
+  group: docker-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    name: Build Docker image
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Set up JDK 17
+        uses: actions/setup-java@v5
+        with:
+          java-version: 17
+          distribution: temurin
+          cache: maven
+
+      - name: Build Wayang distribution
+        run: |
+          ./mvnw clean install -B -Dmaven.test.skip=true
+          ./mvnw package -B -pl :wayang-assembly -Pdistribution -Dmaven.test.skip=true
+
+      - name: Build Docker image
+        run: docker buildx build --load -t apache-wayang:ci .
+
+      - name: Smoke test WordCount on Java platform
+        run: |
+          docker run --rm apache-wayang:ci \
+            org.apache.wayang.apps.wordcount.Main \
+            java \
+            file:///opt/wayang/smoke/wordcount.txt
+
+      - name: Build Python API Docker image
+        run: docker buildx build --load -f python/Dockerfile -t apache-wayang-python:ci .
+
+      - name: Smoke test Python API against REST host
+        run: |
+          set -e
+          docker network create wayang-python-smoke
+          cleanup() {
+            docker rm -f wayang-rest >/dev/null 2>&1 || true
+            docker network rm wayang-python-smoke >/dev/null 2>&1 || true
+          }
+          trap cleanup EXIT
+          docker run -d --name wayang-rest --network wayang-python-smoke apache-wayang-python:ci
+          docker run --rm --network wayang-python-smoke \
+            --entrypoint /usr/local/bin/wayang-python-entrypoint \
+            -e WAYANG_API_HOST=wayang-rest \
+            apache-wayang-python:ci \
+            python python/examples/wordcount.py

Dockerfile

@@ -0,0 +1,45 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM eclipse-temurin:17-jre AS wayang-assembler
+
+ARG WAYANG_DIST=wayang-assembly/target/*-dist.tar.gz
+
+COPY ${WAYANG_DIST} /tmp/wayang-dist.tar.gz
+
+RUN mkdir -p /opt/wayang-root /opt/wayang /opt/wayang/smoke \
+    && tar -xzf /tmp/wayang-dist.tar.gz -C /opt/wayang-root \
+    && extracted_dir="$(find /opt/wayang-root -mindepth 1 -maxdepth 1 -type d -name 'wayang-*' | head -n 1)" \
+    && test -n "${extracted_dir}" \
+    && cp -a "${extracted_dir}/." /opt/wayang/ \
+    && find /opt/wayang -type f \( -name "*-sources.jar" -o -name "*-javadoc.jar" -o -name "README.md" \) -delete \
+    && printf "apache wayang docker smoke test\n" > /opt/wayang/smoke/wordcount.txt \
+    && rm /tmp/wayang-dist.tar.gz
+
+FROM eclipse-temurin:17-jre
+
+ENV WAYANG_HOME=/opt/wayang
+ENV FLAG_WAYANG=true
+ENV PATH="${WAYANG_HOME}/bin:${PATH}"
+
+COPY --from=wayang-assembler /opt/wayang /opt/wayang
+
+RUN mkdir -p /root/.wayang /opt/wayang/conf \
+    && touch /opt/wayang/conf/wayang.properties
+
+WORKDIR /opt/wayang
+
+ENTRYPOINT ["/opt/wayang/bin/wayang-submit"]
+CMD ["org.apache.wayang.apps.pi.PiEstimation", "java", "1"]

bin/wayang-submit

@@ -41,21 +41,6 @@ if [ -z "${WAYANG_HOME}" ]; then
   export WAYANG_HOME=$( get_realpath ${BASH_SOURCE[0]} )
 fi
 
-if [ -z "${SPARK_HOME}" ]; then
-  echo "The variable SPARK_HOME it needs to be setup" >&2
-  exit 1
-fi
-
-#if [ -z "${FLINK_HOME}" ]; then
-#  echo "The variable FLINK_HOME it needs to be setup" >&2
-#  exit 1
-#fi
-
-if [ -z "${HADOOP_HOME}" ]; then
-  echo "The variable HADOOP_HOME it needs to be setup" >&2
-  exit 1
-fi
-
 # Find the java binary
 if [ -n "${JAVA_HOME}" ]; then
   RUNNER="${JAVA_HOME}/bin/java"
@@ -68,57 +53,92 @@ else
   fi
 fi
 
-# Find Spark jars.
-if [ -d "${SPARK_HOME}" ]; then
-  SPARK_JARS_DIR="${SPARK_HOME}/jars"
-fi
+WAYANG_CODE="${WAYANG_HOME}/jars"
 
-# Find Hadoop jars.
-if [ -d "${HADOOP_HOME}" ]; then
-  HADOOP_JARS_DIR="${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*"
-fi
+WAYANG_LIBS="${WAYANG_HOME}/libs"
 
-if [ "$(ls ${SPARK_JARS_DIR} | grep ^hadoop | wc -l)" == "0" ]; then
+WAYANG_CONF="${WAYANG_HOME}/conf"
 
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-common | grep -v tests | head -n 1)"
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-core | grep -v tests | head -n 1)"
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-jobclient | grep -v tests | head -n 1)"
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/$(ls ${HADOOP_HOME}/share/hadoop/hdfs | grep ^hadoop-hdfs-client | grep -v tests | head -n 1)"
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-annotations | grep -v tests | head -n 1)"
-  HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-auth | grep -v tests | head -n 1)"
+# Bootstrap the classpath.
+WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
 
-fi
+append_classpath() {
+  if [ -n "$1" ]; then
+    WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:$1"
+  fi
+}
 
+append_first_matching_jar() {
+  local dir="$1"
+  local pattern="$2"
+  local jar
 
-WAYANG_CODE="${WAYANG_HOME}/jars"
+  if [ -d "${dir}" ]; then
+    jar=$(find "${dir}" -maxdepth 1 -name "${pattern}" ! -name "*tests*" | head -n 1)
+    append_classpath "${jar}"
+  fi
+}
 
-WAYANG_LIBS="${WAYANG_HOME}/libs"
+if [ -n "${SPARK_HOME}" ]; then
+  if [ ! -d "${SPARK_HOME}/jars" ]; then
+    echo "SPARK_HOME is set but ${SPARK_HOME}/jars does not exist" >&2
+    exit 1
+  fi
+  append_classpath "${SPARK_HOME}/jars/*"
+fi
 
-WAYANG_CONF="${WAYANG_HOME}/conf"
+if [ -n "${HADOOP_HOME}" ]; then
+  if [ ! -d "${HADOOP_HOME}/share/hadoop/common" ]; then
+    echo "HADOOP_HOME is set but ${HADOOP_HOME}/share/hadoop/common does not exist" >&2
+    exit 1
+  fi
+  append_classpath "${HADOOP_HOME}/share/hadoop/common/*"
+  append_classpath "${HADOOP_HOME}/share/hadoop/common/lib/*"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-common*.jar"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-core*.jar"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/mapreduce" "hadoop-mapreduce-client-jobclient*.jar"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs" "hadoop-hdfs-client*.jar"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs/lib" "hadoop-annotations*.jar"
+  append_first_matching_jar "${HADOOP_HOME}/share/hadoop/hdfs/lib" "hadoop-auth*.jar"
+fi
 
-# Bootstrap the classpath.
-WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*"
-WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${SPARK_JARS_DIR}/*:${HADOOP_JARS_DIR}"
+if [ -n "${FLINK_HOME}" ]; then
+  if [ ! -d "${FLINK_HOME}/lib" ]; then
+    echo "FLINK_HOME is set but ${FLINK_HOME}/lib does not exist" >&2
+    exit 1
+  fi
+  append_classpath "${FLINK_HOME}/lib/*"
+fi
+
+append_classpath "${WAYANG_EXTRA_CLASSPATH}"
+
+FLAGS=()
+FLAGS+=(
+  "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED"
+  "--add-exports=java.base/jdk.internal.misc=ALL-UNNAMED"
+  "--add-opens=java.base/java.nio=ALL-UNNAMED"
+  "--add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+  "--add-opens=java.base/java.lang=ALL-UNNAMED"
+  "--add-opens=java.base/java.util=ALL-UNNAMED"
+  "--add-opens=java.base/java.io=ALL-UNNAMED"
+  "--add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+  "--add-opens=java.base/sun.reflect.annotation=ALL-UNNAMED"
+  "--add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+  "--add-opens=java.base/java.net=ALL-UNNAMED"
+  "--add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+)
 
-FLAGS=""
 if [ "${FLAG_LOG}" = "true" ]; then
-	FLAGS="${FLAGS} -Dlog4j.configuration=file://${WAYANG_CONF}/log4j.properties"
+	FLAGS+=("-Dlog4j.configuration=file://${WAYANG_CONF}/log4j.properties")
 fi
 
 if [ "${FLAG_WAYANG}" = "true" ]; then
-	FLAGS="${FLAGS} -Dwayang.configuration=file://${WAYANG_CONF}/wayang.properties"
+	FLAGS+=("-Dwayang.configuration=file://${WAYANG_CONF}/wayang.properties")
 fi
 
 if [ -n "${OTHER_FLAGS}" ]; then
-	FLAGS="${FLAGS} ${OTHER_FLAGS}"
+  read -r -a OTHER_FLAGS_ARRAY <<< "${OTHER_FLAGS}"
+	FLAGS+=("${OTHER_FLAGS_ARRAY[@]}")
 fi
 
-# Wrap args in quotes to be able to execute args with parenthesis, spaces, etc
-ARGS=""
-for arg in $(echo ${@:2})
-do
-  ARGS="$ARGS \"${arg}\""
-done
-
-eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${ARGS}"
-
+exec "$RUNNER" "${FLAGS[@]}" -cp "${WAYANG_CLASSPATH}" "$CLASS" "${@:2}"

python/Dockerfile

@@ -0,0 +1,55 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG WAYANG_IMAGE=apache-wayang:ci
+FROM ${WAYANG_IMAGE} AS wayang-runtime
+
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1
+ENV WAYANG_API_HOST=wayang-rest
+ENV WAYANG_API_PORT=8080
+ENV FLAG_WAYANG=true
+ENV JAVA_HOME=/opt/java/openjdk
+ENV WAYANG_HOME=/opt/wayang
+ENV PATH=/opt/wayang/bin:/opt/java/openjdk/bin:/opt/wayang/python/.venv/bin:${PATH}
+
+USER root
+
+COPY --from=wayang-runtime /opt/java/openjdk /opt/java/openjdk
+COPY --from=wayang-runtime /opt/wayang /opt/wayang
+
+WORKDIR /opt/wayang/python
+
+COPY python/ /opt/wayang/python/
+
+RUN mkdir -p /root/.wayang \
+    && python3 -m venv /opt/wayang/python/.venv \
+    && python -m pip install --no-cache-dir -r src/pywy/requirements.txt \
+    && python -m pip install --no-cache-dir . \
+    && printf '%s\n' \
+        'wayang.api.python.worker = /opt/wayang/python/src/pywy/execution/worker.py' \
+        'wayang.api.python.path = /opt/wayang/python/.venv/bin/python' \
+        'wayang.api.python.env.path = /opt/wayang/python/src' \
+        >> /opt/wayang/conf/wayang.properties
+
+COPY python/docker/entrypoint.sh /usr/local/bin/wayang-python-entrypoint
+
+RUN chmod +x /usr/local/bin/wayang-python-entrypoint
+
+WORKDIR /opt/wayang
+
+ENTRYPOINT ["bin/wayang-submit"]
+CMD ["org.apache.wayang.api.json.Main", "8080"]