Dockerfile Spark Application - keshavbaweja-git/guides GitHub Wiki

Dockerfile

FROM keshavbaweja/spark:2.4.4-hadoop2.7

WORKDIR /opt/spark/myapp
COPY ./target/myapp-*-deploy.zip .
RUN unzip myapp-*-deploy.zip -d .
RUN rm ./myapp-*-deploy.zip

WORKDIR /opt/spark/work-dir

Kubernetes Launch Command

/apps/spark-2.4.4-bin-hadoop2.7/bin/spark-submit \
    --master <kubernetes-master-url> \
    --deploy-mode cluster \
    --name <myapp> \
    --class <com.baweja.App> \
    --conf "spark.driver.extraJavaOptions=-Dmyapp.env.name=k8s" \
    --conf spark.kubernetes.container.image=myapp:1.0.0 \
    --conf spark.kubernetes.namespace=<namespace> \
    --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.myapp-spec-pv.options.claimName=myapp-spec-pvc \
    --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.myapp-spec-pv.mount.path=/myapp/folder1 \
    --conf spark.kubernetes.driver.volumes.persistentVolumeClaim.myapp-spec-pv.mount.readOnly=false \
    --conf spark.kubernetes.executor.volumes.persistentVolumeClaim.myapp-spec-pv.options.claimName=myapp-spec-pvc \
    --conf spark.kubernetes.executor.volumes.persistentVolumeClaim.myapp-spec-pv.mount.path=/myapp/folder1 \
    --conf spark.kubernetes.executor.volumes.persistentVolumeClaim.myapp-spec-pv.mount.readOnly=false \
    local:///opt/spark/myapp/lib/myapp.jar "arg1" "arg2"
⚠️ **GitHub.com Fallback** ⚠️