apiVersion: "sparkoperator.hpe.com/v1beta2"
kind: SparkApplication
metadata:
  name: data-process-transfer-demo
  namespace: <username>
spec:
  sparkConf:
    spark.hadoop.fs.s3a.path.style.access: "true"
    spark.hadoop.fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
    spark.hadoop.fs.s3a.connection.ssl.enabled: "false"
    spark.hadoop.fs.s3a.aws.credentials.provider: "org.apache.spark.s3a.EzSparkAWSCredentialProvider"
    spark.hadoop.fs.s3a.endpoint: http://local-s3-service.ezdata-system.svc.cluster.local:30000
  type: Scala
  sparkVersion: 3.5.5
  mode: cluster
  image: lr1-bd-harbor-registry.mip.storage.hpecorp.net/develop/hpe-spark/spark:v3.5.5.2.6
  imagePullPolicy: Always
  mainApplicationFile: "local:///mounts/shared-volume/shared/aie-tutorials/current-release/Data-Analytics/Spark/DataProcessTransfer/DataProcessTransfer.jar"
  mainClass: com.mapr.sparkdemo.DataProcessTransfer
  arguments:
    - s3a://aie-demo/data/financial.csv               # source path
    - csv                                              # source format
    - file:///mounts/shared-volume/shared/financial-processed # destination path
    - parquet                                          # destination format
  restartPolicy:
    type: Never
  imagePullSecrets:
    - imagepull
  driver:
    labels:
      version: 3.5.5
    cores: 1
    coreLimit: "1"
    memory: 1g
  executor:
    labels:
      version: 3.5.5
    cores: 1
    coreLimit: "1"
    memory: 512m
    instances: 1
  volumes:
    - name: shared-volume
      persistentVolumeClaim:
        claimName: kubeflow-shared-pvc
