# Logging to the Access Server (as.abci.ai)
yourpc$ ssh -L 10022:es:22 -l username as.abci.ai -i ~/.ssh/secret key filename
# Leave the terminal as it is while using ABCI.

# Port forwarding to the Interactive Node (es)
yourpc$ ssh -p 10022 -l username localhost

# Uploading file (local-filename) (after loiingin to the Interactive Node, type at a different terminal)
yourpc$ scp -P 10022 local-filename username@localhost:/home/username/

# Downloading file (remote-filename) (after loiingin to the Interactive Node, type at a different terminal)
yourpc$ scp -P 10022 username@localhost:/home/username/remote-filename ./

# Executing an Interactive Job (qrsh) (at the same terminal as logging to the Interactive Node)
[username@es1 ~]$ qrsh -g groupname -l rt_F=1 -l h_rt=01:00:00

# Executing a Butch Job(qsub) (at the same terminal as logging to the Interactive Node)
[username@es1 ~]$ qsub -g groupname -l rt_C.small=1 sample.sh

# Building tensor flow-gpu environment and executing program
# First, execute qrsh
[username@es1 ~]$ qrsh -g groupname -l rt_F=1 -l h_rt=01:00:00

# Then, execute the followings
## Load modules
[username@g0001 ~]$ module load python/3.6/3.6.5 cuda/9.0/ cudnn/7.4/7.4.2

## Get in the python virtual environment
[username@g0001 ~]$ python3 -m venv ~/Sample/v_tf_gpu
[username@g0001 ~]$ source ~/Sample/v_tf_gpu/bin/activate

## Install necessary libraries (only once)
(v_tf_gpu)[username@g0001 ~]$ pip3 uninstall tensorflow-gpu # when old tensorflow-gpu exists, unistall it
(v_tf_gpu)[username@g0001 ~]$ pip3 install --ignore-installed --upgrade https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.12.0-cp36-cp36m-linux_x86_64.whl

## Execute TensorFlow by python
(v_tf_gpu)[username@g0001 ~]$ python
>>> import tensorflow as tf
>>> # use TensorFlow

## get out of python virtual environment(Ctrl-D (i.e. EOF) to exit)
(v_tf_gpu)[username@g0001 ~]$

# Executing mnist by qsub

## Sample script (run_mnist.sh)
# Initialaze virtual environment
source /etc/profile.d/modules.sh
# Load modeules
module load python/3.6/3.6.5 cuda/9.0/ cudnn/7.4/7.4.2
# Activate virtula environment
source ~/Sample/v_tf_gpu/bin/activate
# Execute program
cd ~/Sample/v_tf_gpu
python sample_mnist.py

## Sample program (sample_mnist.py)
import tensorflow as tf
mnist = tf.keras.datasets.mnist

(x_train, y_train),(x_test, y_test) = mnist.load_data()
x_train, x_test = x_train / 255.0, x_test / 255.0

def create_model():
model = tf.keras.models.Sequential([
tf.keras.layers.Flatten(input_shape=(28, 28)),
tf.keras.layers.Dense(512, activation=tf.nn.relu),
tf.keras.layers.Dense(10, activation=tf.nn.softmax)
return model
model = create_model()
model.fit(x_train, y_train, epochs=3)
print(model.evaluate(x_test, y_test))

## Uploading script and program by scp
yourpc$ scp -P 10022 run_mnist.sh aaa12345xx@localhost:run_mnist.sh
yourpc$ scp -P 10022 sample_mnist.py aaa12345xx@localhost:sample_mnist.py

## Commands at the Interactive Node
# Authorise access rights to the script and program
[username@es3 ~]$ chmod u+x run_mnist.sh
[username@es3 ~]$ chmod u+x sample_mnist.py
[username@es3 ~]$ ls –l
-rwxr----- 1 aaa12345xx aaa12345xx     417  3月 27 11:13 run_mnist.sh
-rwxr----- 1 aaa12345xx aaa12345xx     720  3月 27 10:05 sample_mnist.py
# Make sure that it has changed to "-rwxr–"

[username@es3 ~]$ qsub -g gaa12345 -l rt_G.small=1 run_mnist.sh

# How to use Jupyter Notebook

## (1) Login to the Interactive Node (es), and Install Jupyter Notebook.(only once)
[username@es3 ~]$ module load python/3.6/3.6.5
[username@es3 ~]$ python3 -m venv ~/lib/pyenv/jupyter_test
[username@es3 ~]$ source ~/lib/pyenv/jupyter_test/bin/activate
(jupyter_test) es3 $ pip install --upgrade pip
(jupyter_test) es3 $ pip install jupyter
(jupyter_test) es3 $ deactivate

## (2) Activate Jupyter Notebook at the Interactive job (qrsh).
[username@es3 ~]$ qrsh -g gaa12345 -l rt_F=1 -l h_rt=01:00:00
[username@g0019 ~]$ module load python/3.6/3.6.5
# g0019: assigned computational resource
[username@g0019 ~]$ source ~/lib/pyenv/jupyter_test/bin/activate
(jupyter_test) [username@g0019 ~]$ jupyter notebook --no-browser --ip=`hostname` >> jupyter.log 2>&1 &
(jupyter_test) [username@g0019 ~]$ jupyter notebook list
Currently running servers:
http://g0004.abci.local:8888/?token=e7f0ba979d4ffd9eeb7e6debf5a326f853fc289583f92dc5 :: /fs3/home/username

## (3) At a different terminal.
yourpc$ ssh -L 10022:es:22 -l username as.abci.ai

## (4) At another terminal.
yourpc$ ssh -L 18888:g0004:8888 -l username (-i ~/.ssh/id_rsa) -p 10022 localhost
# -i: secret key option (optional)

## (5) Access at a web browser (copy a token at (2))