[Addon kubevela#579] Refactor the spark-workload parameter definition…

… and add spark-py example Signed-off-by: yanghua <[email protected]>
yanghua · Mar 2, 2023 · 7036a63 · 7036a63
1 parent 8916021
commit 7036a63
Show file tree

Hide file tree

Showing 4 changed files with 189 additions and 43 deletions.
diff --git a/examples/spark-kubernetes-operator/sparkapp-py.yaml b/examples/spark-kubernetes-operator/sparkapp-py.yaml
@@ -0,0 +1,49 @@
+apiVersion: core.oam.dev/v1beta1
+kind: Application
+metadata:
+  name: spark-app-v1
+  namespace: spark-cluster
+spec:
+  components:
+  - name: spark-workload-component
+    type: spark-workload
+    properties:
+      name: my-spark-py-app
+      namespace: spark-cluster
+      type: Python
+      pythonVersion: "3"
+      mode: cluster
+      image: "gcr.io/spark-operator/spark-py:v3.1.1"
+      imagePullPolicy: Always
+      mainClass: org.apache.spark.examples.streaming.JavaQueueStream
+      mainApplicationFile: "local:///opt/spark/examples/src/main/python/pi.py"
+      sparkVersion: "3.1.1"
+      restartPolicy:
+        type: OnFailure
+        onFailureRetries: 3
+        onFailureRetryInterval: 10
+        onSubmissionFailureRetries: 5
+        onSubmissionFailureRetryInterval: 20
+      volumes:
+        - name: "test-volume"
+          hostPath:
+            path: "/tmp"
+            type: Directory
+      driver:
+        cores: 1
+        coreLimit: "1200m"
+        memory: "1024m"
+        labels:
+          version: 3.1.1
+        volumeMounts:
+          - name: "test-volume"
+            mountPath: "/tmp"
+      executor:
+        cores: 1
+        instances: 1
+        memory: "1024m"
+        labels:
+          version: 3.1.1
+        volumeMounts:
+          - name: "test-volume"
+            mountPath: "/tmp"
diff --git a/examples/spark-kubernetes-operator/sparkapp.yaml b/examples/spark-kubernetes-operator/sparkapp.yaml
@@ -17,18 +17,28 @@ spec:
       mainClass: org.apache.spark.examples.streaming.JavaQueueStream
       mainApplicationFile: "local:///opt/spark/examples/jars/spark-examples_2.12-3.1.1.jar"
       sparkVersion: "3.1.1"
+      restartPolicy:
+        type: Never
       volumes:
         - name: "test-volume"
           hostPath:
             path: "/tmp"
             type: Directory
       driver:
         cores: 1
+        coreLimit: "1200m"
+        memory: "1024m"
+        labels:
+          version: 3.1.1
         volumeMounts:
           - name: "test-volume"
             mountPath: "/tmp"
       executor:
         cores: 1
+        instances: 1
+        memory: "1024m"
+        labels:
+          version: 3.1.1
         volumeMounts:
           - name: "test-volume"
             mountPath: "/tmp"