apiVersion: "sparkoperator.hpe.com/v1beta2"
kind: SparkApplication
metadata:
  name: spark-fts-{{ts_nodash|replace("T", "")}}
spec:
  sparkConf:
    spark.driver.extraJavaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
    spark.executor.extraJavaOptions: "-Dcom.amazonaws.sdk.disableCertChecking=true"
  hadoopConf:
    fs.s3a.path.style.access: "true"
    fs.s3a.impl: "org.apache.hadoop.fs.s3a.S3AFileSystem"
    fs.s3a.endpoint: '{{dag_run.conf["s3_endpoint"]}}'
    fs.s3a.connection.ssl.enabled: '{{"true" if dag_run.conf["s3_endpoint_ssl_enabled"] else "false"}}'
    fs.s3a.aws.credentials.provider: "org.apache.spark.s3a.EzSparkAWSCredentialProvider"
  type: Scala
  sparkVersion: 3.5.5
  mode: cluster
  image: '{{dag_run.conf["registry_url"]}}hpe-spark/spark:v3.5.5.2.3'
  imagePullPolicy: Always
  mainApplicationFile: local:///mounts/shared-volume/shared/aie-tutorials/current-release/Data-Analytics/Spark/DataProcessTransfer/DataProcessTransfer.jar
  mainClass: com.mapr.sparkdemo.DataProcessTransfer
  arguments:
    - s3a://{{dag_run.conf["s3_bucket"]}}/{{dag_run.conf["s3_path"]}}
    - csv
    - file:///mounts/shared-volume/shared/{{dag_run.conf["export_path"]}}
    - parquet
  restartPolicy:
    type: Never
  driver:
    cores: 1
    coreLimit: "1000m"
    memory: "1g"
    labels:
      version: 3.5.5
    volumeMounts:
      - name: shared-volume
        mountPath: /mounts/shared-volume/shared
  executor:
    cores: 1
    coreLimit: "2000m"
    instances: 2
    memory: "512m"
    labels:
      version: 3.5.5
    volumeMounts:
      - name: shared-volume
        mountPath: /mounts/shared-volume/shared
    volumes:
      - name: shared-volume
        persistentVolumeClaim:
          claimName: kubeflow-shared-pvc
