Skip to main content

TPCDS Spark Benchmark Results for Graviton R6g, R7g, and R8g

This page has the results of our benchmarking tests on R-series Graviton instances, demonstrating up to a 1.6x faster runtime on newer generation instances.

These benchmarks were executed using the steps defined in the Running the Benchmark section across 1TB of data. We used the same EKS cluster, the same data set, the same number of nodes, and the same addons and configuration for all of these tests. We simply changed the instance types that were used for each run. The full configuration details are below:

To view the Managed Node Group configuration, Click to toggle content!
    spark_benchmark_ebs = {
      name        = "spark_benchmark_ebs"
      description = "Managed node group for Spark Benchmarks with EBS using x86 or ARM"
      # Filtering only Secondary CIDR private subnets starting with "100.". Subnet IDs where the nodes/node groups will be provisioned
      subnet_ids = [element(compact([for subnet_id, cidr_block in zipmap(module.vpc.private_subnets, module.vpc.private_subnets_cidr_blocks) :
        substr(cidr_block, 0, 4) == "100." ? subnet_id : null]), 0)
      ]

      # Change ami_type= AL2023_x86_64_STANDARD for x86 instances
      ami_type = "AL2023_ARM_64_STANDARD" # arm64

      # Node group will be created with zero instances when you deploy the blueprint.
      # You can change the min_size and desired_size to 6 instances
      # desired_size might not be applied through terrafrom once the node group is created so this needs to be adjusted in AWS Console.
      min_size     = 0 # Change min and desired to 6 for running benchmarks
      max_size     = 8
      desired_size = 0 # Change min and desired to 6 for running benchmarks

      # This storage is used as a shuffle for non NVMe SSD instances. e.g., r8g instances
      block_device_mappings = {
        xvda = {
          device_name = "/dev/xvda"
          ebs = {
            volume_size           = 300
            volume_type           = "gp3"
            iops                  = 3000
            encrypted             = true
            delete_on_termination = true
          }
        }
      }

      # Change the instance type as you desire and match with ami_type
      instance_types = ["r8g.12xlarge"] # Change Instance type to run the benchmark with various instance types

      labels = {
        NodeGroupType = "spark_benchmark_ebs"
      }

      tags = {
        Name          = "spark_benchmark_ebs"
        NodeGroupType = "spark_benchmark_ebs"
      }
    }
To view the Spark Application configuration, Click to toggle content!
apiVersion: "sparkoperator.k8s.io/v1beta2"
kind: SparkApplication
metadata:
  name: tpcds-benchmark-1tb-ebs # Change for each test with instancetype etc,
  namespace: spark-team-a
spec:
  # Temporarily commented out until the YuniKorn issue is resolved; falls back to the default Kubernetes scheduler
  # batchScheduler: yunikorn
  # batchSchedulerOptions:
  #   queue: root.default
  type: Scala
  mode: cluster
  image: public.ecr.aws/data-on-eks/spark3.5.3-scala2.12-java17-python3-ubuntu-tpcds:v2
  imagePullPolicy: IfNotPresent
  sparkVersion: 3.5.3
  mainClass: com.amazonaws.eks.tpcds.BenchmarkSQL
  mainApplicationFile: local:///opt/spark/examples/jars/eks-spark-benchmark-assembly-1.0.jar
  arguments:
    # TPC-DS data location
    - "s3a://<S3_BUCKET>/TPCDS-TEST-1TB"
    # results location
    - "s3a://<S3_BUCKET>/TPCDS-TEST-1T-RESULT"
    # Path to kit in the docker image
    - "/opt/tpcds-kit/tools"
    # Data Format
    - "parquet"
    # Scale factor (in GB)
    - "1000" # changed from 3000 to 100gb for demo
    # Number of iterations
    - "1"
    # Optimize queries with hive tables
    - "false"
    # Filter queries, will run all if empty - "q98-v2.4,q99-v2.4,ss_max-v2.4,q95-v2.4"
    - ""
    # Logging set to WARN
    - "true"
  sparkConf:
    # Expose Spark metrics for Prometheus
    "spark.ui.prometheus.enabled": "true"
    "spark.executor.processTreeMetrics.enabled": "true"
    "spark.metrics.conf.*.sink.prometheusServlet.class": "org.apache.spark.metrics.sink.PrometheusServlet"
    "spark.metrics.conf.driver.sink.prometheusServlet.path": "/metrics/driver/prometheus/"
    "spark.metrics.conf.executor.sink.prometheusServlet.path": "/metrics/executors/prometheus/"

    # Spark Event logs
    "spark.eventLog.enabled": "true"
    "spark.eventLog.dir": "s3a://<S3_BUCKET>/spark-event-logs"
    "spark.eventLog.rolling.enabled": "true"
    "spark.eventLog.rolling.maxFileSize": "64m"

    "spark.network.timeout": "2000s"
    "spark.executor.heartbeatInterval": "300s"
    # AQE
    "spark.sql.adaptive.enabled": "true"
    "spark.sql.adaptive.localShuffleReader.enabled": "true"
    "spark.sql.adaptive.coalescePartitions.enabled": "true"
    "spark.sql.adaptive.skewJoin.enabled": "true"
    "spark.kubernetes.executor.podNamePrefix": "benchmark-exec-ebs"
   # S3 Optimizations
    # "spark.hadoop.fs.s3a.aws.credentials.provider": "com.amazonaws.auth.WebIdentityTokenCredentialsProvider" # This is using AWS SDK V1 in maintenance mode
    "spark.hadoop.fs.s3a.aws.credentials.provider.mapping": "com.amazonaws.auth.WebIdentityTokenCredentialsProvider=software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"
    "spark.hadoop.fs.s3a.aws.credentials.provider": "software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider"  # AWS SDK V2 https://hadoop.apache.org/docs/stable/hadoop-aws/tools/hadoop-aws/aws_sdk_upgrade.html
    "spark.hadoop.fs.s3.impl": "org.apache.hadoop.fs.s3a.S3AFileSystem"
    "spark.hadoop.fs.s3a.fast.upload": "true"
    "spark.hadoop.fs.s3a.path.style.access": "true"
    "spark.hadoop.fs.s3a.fast.upload.buffer": "disk"
    "spark.hadoop.fs.s3a.buffer.dir": "/tmp/s3a"
    "spark.hadoop.fs.s3a.multipart.size": "128M" # Good for large files
    "spark.hadoop.fs.s3a.multipart.threshold": "256M"
    "spark.hadoop.fs.s3a.threads.max": "50"
    "spark.hadoop.fs.s3a.connection.maximum": "200"

    "spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version": "2"
    "spark.executor.defaultJavaOptions": "-verbose:gc -XX:+UseParallelGC -XX:InitiatingHeapOccupancyPercent=70"
    # "spark.hadoop.fs.s3a.readahead.range": "256K"

    # -----------------------------------------------------
    # This block is very critical when you get errors like
    #     Exception in thread \"main\" io.fabric8.kubernetes.client.KubernetesClientException: An error has occurred
    #     Caused by: java.net.SocketTimeoutException: timeout
    # spark.kubernetes.local.dirs.tmpfs: "true" # More details here https://spark.apache.org/docs/latest/running-on-kubernetes.html#using-ram-for-local-storage
    spark.kubernetes.submission.connectionTimeout: "120000" # milliseconds
    spark.kubernetes.submission.requestTimeout: "120000"
    spark.kubernetes.driver.connectionTimeout: "120000"
    spark.kubernetes.driver.requestTimeout: "120000"
    # spark.kubernetes.allocation.batch.size: "20" # default 5 but adjust according to your cluster size
    # -----------------------------------------------------
    # S3 Optimizations
    "spark.hadoop.fs.s3a.multipart.size": "67108864"           # 64 MB part size for S3 uploads
    "spark.hadoop.fs.s3a.threads.max": "40"                     # Limit S3 threads for optimized throughput
    "spark.hadoop.fs.s3a.connection.maximum": "100"             # Set max connections for S3

    # Data writing and shuffle tuning
    "spark.shuffle.file.buffer": "1m"                           # Increase shuffle buffer for better disk I/O
    "spark.reducer.maxSizeInFlight": "48m"                      # Increase reducer buffer size in-flight data

    # Optional: Tuning multipart upload threshold
    "spark.hadoop.fs.s3a.multipart.purge": "true"               # Automatically clear failed multipart uploads
    "spark.hadoop.fs.s3a.multipart.threshold": "134217728"      # 128 MB threshold to start multi-part upload
  driver:
    cores: 5
    memory: "20g"
    memoryOverhead: "6g"
    serviceAccount: spark-team-a
    securityContext:
      runAsUser: 185
    env:
      - name: JAVA_HOME
        value: "/opt/java/openjdk"
    nodeSelector:
      NodeGroupType: spark_benchmark_ebs
  executor:
    cores: 5
    memory: "20g"
    memoryOverhead: "6g"
    # 8 executors per node
    instances: 36 # 6 pods per node; 6 nodes with EKS Managed Node group
    serviceAccount: spark-team-a
    securityContext:
      runAsUser: 185
    env:
      - name: JAVA_HOME
        value: "/opt/java/openjdk"
    nodeSelector:
      NodeGroupType: spark_benchmark_ebs
  restartPolicy:
    type: Never
To view the Dockerfile for the Spark container image, Click to toggle content!
# Use the official Spark base image with Java 17 and Python 3
FROM apache/spark:3.5.3-scala2.12-java17-python3-ubuntu

# Arguments for version control
ARG HADOOP_VERSION=3.4.1
ARG AWS_SDK_VERSION=2.29.0
ARG SPARK_UID=185

# Set environment variables
ENV SPARK_HOME=/opt/spark

# Set up as root to install dependencies and tools
USER root

# Install necessary build tools and specific sbt version 0.13.18
RUN apt-get update && \
    apt-get install -y \
        gcc \
        make \
        flex \
        bison \
        git \
        openjdk-17-jdk \
        wget \
        curl && \
    # Install sbt 0.13.18
    wget https://github.com/sbt/sbt/releases/download/v0.13.18/sbt-0.13.18.tgz && \
    tar -xzf sbt-0.13.18.tgz -C /usr/local && \
    ln -s /usr/local/sbt/bin/sbt /usr/local/bin/sbt && \
    # Cleanup
    rm sbt-0.13.18.tgz && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

# Clone and compile TPC-DS toolkit
WORKDIR /opt
RUN git clone https://github.com/databricks/tpcds-kit.git && \
    cd tpcds-kit/tools && \
    make OS=LINUX && \
    chmod +x dsdgen dsqgen

# Clone the SQL perf library and related files
# Change the branch from delta to tpcds-v2.13 for latest
RUN git clone -b delta https://github.com/aws-samples/emr-on-eks-benchmark.git /tmp/emr-on-eks-benchmark

# Build the Databricks SQL perf library
RUN cd /tmp/emr-on-eks-benchmark/spark-sql-perf && sbt +package

# Use the compiled Databricks SQL perf library to build benchmark utility
RUN cd /tmp/emr-on-eks-benchmark/ && \
    mkdir -p /tmp/emr-on-eks-benchmark/benchmark/libs && \
    cp /tmp/emr-on-eks-benchmark/spark-sql-perf/target/scala-2.12/*.jar /tmp/emr-on-eks-benchmark/benchmark/libs && \
    cd /tmp/emr-on-eks-benchmark/benchmark && sbt assembly

# Remove any old Hadoop libraries
RUN rm -f ${SPARK_HOME}/jars/hadoop-client-* && \
    rm -f ${SPARK_HOME}/jars/hadoop-yarn-server-web-proxy-*.jar

# Add Hadoop AWS connector and AWS SDK for S3A support, along with hadoop-common dependencies
# TODO: hadoop-common, hadoop-yarn-server-web-proxy might not be required. Remove these and test it.
RUN cd ${SPARK_HOME}/jars && \
    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/${HADOOP_VERSION}/hadoop-aws-${HADOOP_VERSION}.jar && \
    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-api/${HADOOP_VERSION}/hadoop-client-api-${HADOOP_VERSION}.jar && \
    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-client-runtime/${HADOOP_VERSION}/hadoop-client-runtime-${HADOOP_VERSION}.jar && \
    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-common/${HADOOP_VERSION}/hadoop-common-${HADOOP_VERSION}.jar && \
    wget https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-yarn-server-web-proxy/${HADOOP_VERSION}/hadoop-yarn-server-web-proxy-${HADOOP_VERSION}.jar && \
    wget https://repo1.maven.org/maven2/software/amazon/awssdk/bundle/${AWS_SDK_VERSION}/bundle-${AWS_SDK_VERSION}.jar

# Create directory for TPC-DS data and set permissions
RUN mkdir -p /opt/tpcds-data && \
    chown -R ${SPARK_UID}:${SPARK_UID} /opt/tpcds-data

# Copy the built JARs to Spark's jars directory
RUN mkdir -p ${SPARK_HOME}/examples/jars/ && \
    cp /tmp/emr-on-eks-benchmark/benchmark/target/scala-2.12/*jar ${SPARK_HOME}/examples/jars/ && \
    chown -R ${SPARK_UID}:${SPARK_UID} ${SPARK_HOME}/examples

# Set working directory
WORKDIR ${SPARK_HOME}

# Switch to non-root user
USER ${SPARK_UID}

Results​

When reviewing the results for the TPCDS benchmark we are interested in the time it takes for the Spark SQL queries to complete, the faster those queries complete the better. The graph below shows the cumulative runtime in seconds for all of the queries for each instance type we tested:

Total runtimes for the benchmarks per instance type

We can similarly display the time for each query per instance, you can see the improvements in runtime for the newer generations.:

Total runtimes for the benchmarks per instance type

In the table below we have taken the Median times from the output for each instance type from a benchmark with 3 iterations of the queries and calculated the performance gained. You can view the raw output data in the raw_data folder here.

To calculate the performance increase we are calculating a ratio of the query times. For example, to determine how much faster the r8g instances were compared to the r6g instances:

  • Find the times corresponding to each query, using q20-v2.4 as an example the r6g.12xlarge took 2.81s and the r8g.12xlarge took 1.69s.
  • We then divide r5g.12xlarge/r8g.12xlarge, for q20-v2.4 thats 2.81s/1.69s = 1.66. So for this query the r8g.12xlarge was able to complete the queries 1.66 times faster (or a ~66% percent improvement)

The data has been sorted by the last column, showing the performance increase r8g.12xlarge has over the r6g.12xlarge.

Queryr6g.12xlarger7g.12xlarger8g.12xlarger7g times faster than r6gr8g times faster than r7gr8g times faster than r6g
q20-v2.42.811.941.691.451.141.66
q39b-v2.45.754.263.561.351.201.61
q45-v2.47.665.694.921.351.151.56
q8-v2.46.775.194.371.311.191.55
q73-v2.45.083.963.311.281.201.54
q39a-v2.46.604.844.361.361.111.51
q81-v2.422.4120.1414.931.111.351.50
q69-v2.47.815.745.261.361.091.48
q34-v2.45.804.763.961.221.201.47
q97-v2.419.7615.1313.601.311.111.45
q95-v2.448.1438.2433.141.261.151.45
q22-v2.49.797.576.811.291.111.44
q24b-v2.471.0959.1249.641.201.191.43
q15-v2.47.626.305.341.211.181.43
q58-v2.45.054.123.571.221.161.41
q18-v2.410.438.457.441.231.141.40
q14b-v2.453.7340.8838.331.311.071.40
q72-v2.422.2917.1916.081.301.071.39
q98-v2.43.783.262.731.161.201.39
q46-v2.413.3711.009.651.211.141.39
q83-v2.42.391.951.731.221.121.38
q6-v2.411.118.838.081.261.091.37
q31-v2.412.1911.128.871.101.251.37
q11-v2.423.9519.3017.721.241.091.35
q29-v2.418.9115.2914.001.241.091.35
q61-v2.45.264.623.921.141.181.34
q91-v2.43.633.052.711.191.121.34
q5-v2.420.5716.7215.421.231.081.33
q54-v2.46.615.614.991.181.121.32
q23b-v2.4152.46130.94115.371.161.141.32
q51-v2.411.909.909.061.201.091.31
q57-v2.47.626.195.811.231.061.31
q10-v2.48.386.706.411.251.051.31
q24a-v2.475.0763.2457.781.191.091.30
q64-v2.464.4853.9249.691.201.091.30
q3-v2.43.392.442.621.390.931.29
q14a-v2.461.1850.1247.371.221.061.29
q65-v2.417.3014.6513.491.181.091.28
q17-v2.46.955.545.441.251.021.28
q79-v2.45.124.144.011.241.031.28
q47-v2.48.457.566.621.121.141.28
q60-v2.44.303.723.381.161.101.27
ss_max-v2.410.489.138.291.151.101.26
q35-v2.415.9013.9612.651.141.101.26
q68-v2.47.936.646.321.191.051.25
q77-v2.42.391.951.921.231.021.25
q75-v2.438.2732.6330.711.171.061.25
q42-v2.42.191.961.761.111.111.24
q25-v2.45.644.844.551.171.061.24
q93-v2.4160.84142.63130.451.131.091.23
q38-v2.416.8014.7713.661.141.081.23
q74-v2.419.5716.7315.971.171.051.23
q82-v2.414.1012.6611.551.111.101.22
q4-v2.455.3649.7045.471.111.091.22
q23a-v2.4116.74101.4595.941.151.061.22
q30-v2.415.6915.5612.911.011.211.22
q94-v2.420.8717.8717.181.171.041.21
q71-v2.43.833.083.161.240.981.21
q86-v2.43.192.892.641.101.101.21
q26-v2.45.945.674.911.051.151.21
q59-v2.415.7314.3513.031.101.101.21
q27-v2.47.156.265.931.141.061.21
q41-v2.41.241.091.031.141.061.20
q36-v2.46.155.275.111.171.031.20
q56-v2.44.003.463.321.151.041.20
q87-v2.415.4614.1312.961.091.091.19
q21-v2.42.312.111.941.091.081.19
q32-v2.42.041.831.731.111.061.18
q78-v2.482.4375.7070.261.091.081.17
q88-v2.440.3936.3834.441.111.061.17
q48-v2.47.897.246.741.091.071.17
q33-v2.44.063.673.471.111.061.17
q99-v2.49.368.368.021.121.041.17
q16-v2.428.3524.7224.411.151.011.16
q12-v2.42.602.642.240.981.181.16
q28-v2.441.4737.5135.841.111.051.16
q76-v2.423.7221.5520.661.101.041.15
q1-v2.46.245.335.431.170.981.15
q37-v2.47.817.186.851.091.051.14
q90-v2.49.918.978.771.101.021.13
q80-v2.422.6320.7220.131.091.031.12
q66-v2.48.097.547.231.071.041.12
q67-v2.4145.51140.91130.011.031.081.12
q7-v2.47.826.617.001.180.941.12
q50-v2.471.7170.0664.461.021.091.11
q19-v2.44.613.984.161.160.961.11
q89-v2.44.714.534.271.041.061.10
q13-v2.48.758.307.981.051.041.10
q63-v2.44.514.364.121.031.061.10
q85-v2.414.9813.8613.721.081.011.09
q70-v2.48.208.287.550.991.101.09
q62-v2.48.978.608.281.041.041.08
q44-v2.419.2418.9417.781.021.071.08
q84-v2.49.729.519.011.021.051.08
q96-v2.48.928.268.291.081.001.08
q9-v2.434.4632.4932.071.061.011.07
q2-v2.414.1213.0513.211.080.991.07
q52-v2.42.001.971.871.011.051.07
q43-v2.44.394.144.131.061.001.06
q92-v2.41.531.561.460.981.071.05
q40-v2.412.3311.4311.821.080.971.04
q49-v2.426.0126.0225.041.001.041.04
q53-v2.44.534.644.420.981.051.03
q55-v2.42.362.152.501.100.860.94