Environment

dnf install -y numactl libxcrypt-compat.aarch64 perf
mkdir -p /home/amptest
tar xf spec2017_ampere_gcc13_fedora36_siryn_213.tgz -C /home/amptest
cd /home/amptest/ampere_spec2017/spec2017
./install.sh #type "yes" here
cd /home/amptest/ampere_spec2017/

./high_perf.sh

#replace 500.perlbench_r with intrate if you want to run all intrate cases
#replace iterations if you find that the results are different every time.
./run_spec2017.sh --iterations 1 --copies 4 --nobuild --action run 500.perlbench_r

#to find all sub cases
cat /home/amptest/ampere_spec2017/spec2017/benchspec/CPU/intrate.bset

#find result
export SPEC_RESULT_DIR=/home/amptest/ampere_spec2017/spec2017/result
cat ${SPEC_RESULT_DIR}/CPU2017.$(cat ${SPEC_RESULT_DIR}/lock.CPU2017)*.txt

Host test

Fedora38 on Ampere One

uname -a
Linux eng10sys-r119.scc-lab.amperecomputing.com 6.4.0+ #28 SMP PREEMPT_DYNAMIC Sun Sep 10 23:44:46 PDT 2023 aarch64 GNU/Linux

getconf PAGESIZE
4096

Ubuntu22 on Ampere Altra

uname -a
Linux adam-ubuntu 6.3.0-rc4 #9 SMP Fri Apr 21 07:14:57 UTC 2023 aarch64 aarch64 aarch64 GNU/Linux

getconf PAGESIZE
65536

VM Environment

Ampere One

wget https://download.fedoraproject.org/pub/fedora/linux/releases/38/Server/aarch64/images/Fedora-Server-38-1.6.aarch64.raw.xz
xz -dk Fedora-Server-38-1.6.aarch64.raw.xz
qemu-img resize Fedora-Server-38-1.6.aarch64.raw -f raw +80G

#create new primary partition.
fdisk /dev/vda            
    n primary
    t 4 8e
    w
partprobe

#Extend the volume of logical partition
pvcreate /dev/vda4        
pvs
vgextend fedora /dev/vda4
df -h
lvextend -l +100%FREE /dev/mapper/fedora-root
xfs_growfs /dev/mapper/fedora-root

git clone git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
git checkout tags/v6.4 -b 6.4_kernel

copy config-6.4.0 from host to .config
make olddefconfig
make -j 100
make INSTALL_MOD_STRIP=1 modules_install
make install
dracut --force
grub2-mkconfig -o /boot/grub2/grub.cfg
grubby --set-default /boot/vmlinuz-6.4.0

Ampere Altra

wget https://cloud-hypervisor.azureedge.net/jammy-server-cloudimg-arm64-custom-20220329-0.raw
qemu-img resize jammy-server-cloudimg-arm64-custom-20220329-0.raw -f raw +80G

On VM:
git clone git://git.kernel.org/pub/scm/linux/kernel/git/stable/linux-stable.git
git checkout v6.3-rc4

copy config-6.3.0 from host to .config
make olddefconfig
make -j 100
make INSTALL_MOD_STRIP=1 modules_install
make install

Qemu test

Run VM

qemu-system-aarch64 \
        -nographic \
        -qmp unix:/tmp/qmp-test2,server,nowait \
        -machine virt -cpu host -accel kvm \
        -bios images/QEMU_EFI.fd \
        -net user,hostfwd=tcp::2222-:22 -net nic \
        -smp cpus=40 \
        -m 128G \
        -drive if=none,file=ubuntu22.04.raw,format=raw,id=hd1 -device virtio-blk-pci,drive=hd1,bootindex=0 \
        2>&1 | tee log.txt

CPU affinity

python cpu_affinity.py -s /tmp/qmp-test2 {1..40}

cpu_affinity.py

#!/usr/bin/python
# QEMU vCPU pinning tool
#
# Copyright (C) 2016 Red Hat Inc.
#
# Authors:
#  Maxime Coquelin <[email protected]>
#
# This work is licensed under the terms of the GNU GPL, version 2.  See
# the COPYING file in the top-level directory
import argparse
import json
import os

from subprocess import call
from qmp import QEMUMonitorProtocol

pinned = []

parser = argparse.ArgumentParser(description='Pin QEMU vCPUs to physical CPUs')
parser.add_argument('-s', '--server', type=str, required=True,
                    help='QMP server path or address:port')
parser.add_argument('cpu', type=int, nargs='+',
                    help='Physical CPUs IDs')
args = parser.parse_args()

devnull = open(os.devnull, 'w')

srv = QEMUMonitorProtocol(args.server)
srv.connect()

for vcpu in srv.command('query-cpus-fast'):
    vcpuid = vcpu['props']['core-id']
    tid = vcpu['thread-id']
    if tid in pinned:
        print('vCPU{}\'s tid {} already pinned, skipping'.format(vcpuid, tid))
        continue

    cpuid = args.cpu[vcpuid % len(args.cpu)]
    print('Pin vCPU {} (tid {}) to physical CPU {}'.format(vcpuid, tid, cpuid))
    try:
        call(['taskset', '-pc', str(cpuid), str(tid)], stdout=devnull)
        pinned.append(tid)
    except OSError:
        print('Failed to pin vCPU{} to CPU{}'.format(vcpuid, cpuid))

Cloud-hypervisor test

Run VM

# The MAC address must be attached to the macvtap and be used inside the guest
mac="c2:67:4f:53:29:cb"
# Host network adapter to bridge the guest onto
host_net="enP9p3s0"

# Create the macvtap0 as a new virtual MAC associated with the host network
sudo ip link add link "$host_net" name macvtap0 type macvtap
sudo ip link set macvtap0 address "$mac" up
sudo ip link show macvtap0

# A new character device is created for this interface
tapindex=$(< /sys/class/net/macvtap0/ifindex)
tapdevice="/dev/tap$tapindex"

# Ensure that we can access this device
sudo chown "$UID.$UID" "$tapdevice"

#cp /root/workloads/focal-server-cloudimg-arm64-custom-20210929-0.raw /root/workloads/osdisk.img
/root/workloads/cloud-hypervisor-static-aarch64 \
        --cpus boot=40,affinity= 
[0@[1],1@[2],2@[3],3@[4],4@[5],5@[6],6@[7],7@[8],8@[9],9@[10],10@[11],11@[12],12@[13],13@[14],14@[15],15@[16],16@[17],17@[18],18@[19],19@[20],20@[21],21@[22],22@[23],23@[24],24@[25],25@[26],26@[27],27@[28],28@[29],29@[30],30@[31],31@[32],32@[33],33@[34],34@[35],35@[36],36@[37],37@[38],38@[39],39@[40]] \
        --memory size=128G \
        --kernel /root/workloads/CLOUDHV_EFI.fd \
        --disk path=/home/dom/images/ubuntu22.04.raw \
        --vsock cid=3,socket=/tmp/vsock \
        --api-socket /tmp/ch0 \
        --event-monitor path=/tmp/event.json \
        --net fd=3,mac=$mac 3<>$"$tapdevice"

CPU affinity

--cpus boot=40,affinity= 
[0@[1],1@[2],2@[3],3@[4],4@[5],5@[6],6@[7],7@[8],8@[9],9@[10],10@[11],11@[12],12@[13],13@[14],14@[15],15@[16],16@[17],17@[18],18@[19],19@[20],20@[21],21@[22],22@[23],23@[24],24@[25],25@[26],26@[27],27@[28],28@[29],29@[30],30@[31],31@[32],32@[33],33@[34],34@[35],35@[36],36@[37],37@[38],38@[39],39@[40]]

Parse the result:

concat()
{
        num=1
        str=`grep $1 $2 | grep "refrate(ref)" | awk -F , '{print $1"\t"$2"\t"$3"\t"$4}'`
        grep $1 $2 | grep "refrate(ref)" | awk -F , '{print $1"\t"$2"\t"$3"\t"$4}' | while read line;
        do
                echo ${num}_${line} | awk '{print $1"\t"$2"\t"$3"\t"$4}'
                echo ${num}_${line} | awk '{print $1"\t"$2"\t"$3"\t"$4}' >> $3
                let num=$num+1
        done
        if [ "$str" == "" ];then
                echo -e "1_$1\n2_$1\n3_$1" >> $3
        fi
}

files=`find $1 -iname "*.csv"`
rm -rf $1/res.txt
for file in $files;
do
        #grep -iE "500.perlbench_r|502.gcc_r|505.mcf_r|520.omnetpp_r|523.xalancbmk_r|525.x264_r|531.deepsjeng_r|541.leela_r|548.exchange2_r|557.xz_r" $file | grep -E "refrate" | awk -F , '{print $1"\t"$2"\t"$3"\t"$4}' >> $1/res.txt
        concat "500.perlbench_r" $file $1/res.txt
        concat "502.gcc_r" $file $1/res.txt
        concat "505.mcf_r" $file $1/res.txt
        concat "520.omnetpp_r" $file $1/res.txt
        concat "523.xalancbmk_r" $file $1/res.txt
        concat "525.x264_r" $file $1/res.txt
        concat "531.deepsjeng_r" $file $1/res.txt
        concat "541.leela_r" $file $1/res.txt
        concat "548.exchange2_r" $file $1/res.txt
        concat "557.xz_r" $file $1/res.txt
        grep -iE "SPECrate2017_int_base|SPECrate2017_int_peak" $file | awk -F , '{print $1"\t\t\t"$2}' >> $1/res.txt
        echo "" >> $1/res.txt
done

echo "Output result $1/res.txt"