Nuvo-9166GC

DIP Switch on 9000 PCBA

All Off

Testing Environment

dvt@dvt-Nuvo-9000-Series:~$ lsb_release -a
No LSB modules are available.
Distributor ID:	Ubuntu
Description:	Ubuntu 22.04.1 LTS
Release:	22.04
Codename:	jammy

Install GPU Driver

# Need at least 525

# sudo add-apt-repository ppa:graphics-drivers/ppa
# sudo apt install nvidia-driver-525 nvidia-dkms-525
sudo apt install nvidia-driver-525 

sudo reboot

# Get GPU info
nvidia-smi --query-gpu=gpu_name,gpu_bus_id,vbios_version,pcie.link.gen.max,pcie.link.gen.current --format=csv

Get Thermal Sensor Info and Fan Control

sudo apt-get install lm-sensors

sudo sensors-detect
# Except the final one, press Enter all the way down 

## To load everything that is needed, add this to /etc/modules:
## #----cut here----
## # Chip drivers
## coretemp
## it87
## #----cut here----
## If you have some drivers built into your kernel, the list above will
## contain too many modules. Skip the appropriate ones!
## Do you want to add these lines automatically to /etc/modules? (yes/NO)

# YES 


sudo reboot 

# Get Sensor Information
sensors

## coretemp-isa-0000
## Adapter: ISA adapter
## Package id 0:  +35.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 0:        +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 4:        +31.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 8:        +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 12:       +31.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 16:       +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 20:       +31.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 24:       +31.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 28:       +31.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 36:       +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 37:       +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 38:       +32.0°C  (high = +80.0°C, crit = +100.0°C)
## Core 39:       +32.0°C  (high = +80.0°C, crit = +100.0°C)

## it8786-isa-0290
## Adapter: ISA adapter
## in0:         792.00 mV (min =  +1.12 V, max =  +2.62 V)  ALARM
## in1:           1.25 V  (min =  +2.46 V, max =  +0.96 V)  ALARM
## in2:           3.06 V  (min =  +0.35 V, max =  +0.17 V)  ALARM
## +3.3V:         3.65 V  (min =  +3.34 V, max =  +1.82 V)  ALARM
## in4:         552.00 mV (min =  +0.42 V, max =  +0.55 V)  ALARM
## in5:           2.33 V  (min =  +0.32 V, max =  +2.88 V)
## in6:         420.00 mV (min =  +2.88 V, max =  +2.83 V)  ALARM
## 3VSB:          3.65 V  (min =  +2.86 V, max =  +4.99 V)
## Vbat:        144.00 mV
## fan1:        4927 RPM  (min =   34 RPM)
## temp1:        +35.0°C  (low  =  -3.0°C, high = +58.0°C)  sensor = thermistor
## temp2:        +35.0°C  (low  = +101.0°C, high = +31.0°C)  ALARM  sensor = thermistor
## temp3:        +27.0°C  (low  = +110.0°C, high = +11.0°C)  ALARM  sensor = thermistor
## intrusion0:  ALARM


# Fan Control
## hwmon1 willo only be created after lm-sensors installed

sudo su
sudo echo "1" >  /sys/class/hwmon/hwmon1/pwm1_enable

# Full Speed
sudo echo "255" > /sys/class/hwmon/hwmon1/pwm1

# Low Speed
sudo echo "10" > /sys/class/hwmon/hwmon1/pwm1

Run TensorFlow in Docker

Preparation

# Install NVIDIA Container Toolkit to Run Docker with GPU
## Ref: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#docker
# sudo apt install -y curl

# distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
#      && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
#      && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
#            sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
#            sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
            
#sudo apt-get update
#sudo apt-get install -y nvidia-container-toolkit

# Install Docker to run TensorFlow Benchmark in docker
# sudo apt install -y docker.io

#=============================================
#If the method above did not work, run the commands in this block.
### Install Docker ###
sudo apt install -y curl
curl -fsSL https://get.docker.com -o get-docker.sh
sudo sh get-docker.sh
### sudo docker info ###

### Install NVIDAIA Docker container ###
distribution=$(. /etc/os-release;echo $ID$VERSION_ID) \
      && curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \
      && curl -s -L https://nvidia.github.io/libnvidia-container/$distribution/libnvidia-container.list | \
            sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' | \
            sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list
            
sudo apt-get update
sudo apt-get install -y nvidia-container-toolkit

### Bind docker with GPU support ###
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

#=============================================

DOCKER_VERSION=23.03-tf1-py3

sudo docker pull nvcr.io/nvidia/tensorflow:$DOCKER_VERSION

# Bind docker with GPU support
sudo nvidia-ctk runtime configure --runtime=docker
sudo systemctl restart docker

# Download TensorFlow Benchmark on Host
cd ~
git clone https://github.com/tensorflow/benchmarks.git

cd ~/benchmarks/
git checkout cnn_tf_v1.15_compatible 

# git branch
# * cnn_tf_v1.15_compatible
#  master

Run

# Run Docker with Host's TensorFlow Benchmark Folder (benchmarks/scripts/tf_cnn_benchmarks)
DOCKER_VERSION=23.03-tf1-py3

sudo docker run --shm-size=2g --gpus all -it --rm -v $HOME/benchmarks/scripts/tf_cnn_benchmarks:/workspace nvcr.io/nvidia/tensorflow:$DOCKER_VERSION

## Check TensorFlow version in docker
# pip show tensorflow

## Training + Inference
python3 tf_cnn_benchmarks.py --num_gpus=1 --model=resnet50 --variable_update=parameter_server --num_batches=1000000000 --forward_only=true
python3 tf_cnn_benchmarks.py --num_gpus=1 --model=resnet50 --variable_update=parameter_server --num_batches=1000000000 --forward_only=true --use_fp16=true

## Only Inference
python3 tf_cnn_benchmarks.py --num_gpus=1 --model=resnet50 --variable_update=parameter_server --num_batches=1000000000 --forward_only=true --freeze_when_forward_only=true --trt_mode=FP16
python3 tf_cnn_benchmarks.py --num_gpus=1 --model=resnet50 --variable_update=parameter_server --num_batches=1000000000 --forward_only=true --freeze_when_forward_only=true --trt_mode=FP32
python3 tf_cnn_benchmarks.py --num_gpus=1 --model=resnet50 --variable_update=parameter_server --num_batches=1000000000 --forward_only=true --freeze_when_forward_only=true --trt_mode=INT8

## Stress
python3 tf_cnn_benchmarks.py --num_gpus=1 --num_batches=10000000 --batch_size=64 --model=resnet50 --variable_update=parameter_server

Basic Docker Commands

# https://www.mygreatlearning.com/blog/top-essential-docker-commands/

## Get Container ID, e.g., 0a35a0371970
sudo docker ps

## Restart ID
sudo docker restart <ContainerID>
# sudo docker restart 0a35a0371970

## Open another screen with specific container ID
sudo docker exec -ti <ContainerID> /bin/bash
# sudo docker exec -ti 0a35a0371970 /bin/bash