1. はじめに

OpenShift環境は複数ノード必要になり、環境準備などに敷居が高いことが多い。IBMのDeveloperサイトに、AnsibleでSNO(single-node Red Hat OpenShift cluster)をAnsibleでIBM Cloud VPC上の１台のVSI上にプロビジョニングしてくれるチュートリアルが存在したので、今回はその手順に従って試してみた。

2. 完成図

まず、最初にこのチュートリアルに従って作成した環境が最終的にどういう構成になるのかを、記載しておく。

IBM Cloud上に新たにVPCが作成される。
そのVPC上にVSI(Rocky Linux 8: 16 vCPU/64 GiB RAM)が1台作成される。
このVSI上に、２台のKVM Guestが作成される。
- DHCP/DNS Forwarderを実施するFedoraベースのUtilityサーバー
- RHCOSベースのOpenShift Container Platformサーバー
ネットワーク構成は以下の通り

IPアドレス構成は以下の通り
- xx.xx.xx.xx: このVSIに付与されるFloating IP(Global IPアドレス）。
- 10.240.0.xx: eth0に付与されるIPアドレス
- 192.168.122.1: 仮想ブリッジ(birbr0)に付与されるIPアドレス
- 192.168.122.2: Utility Server(DHCP/DNS forwarder)。Fedoraで構成されるKVM Guest
- 192.168.122.3: RHCOS上に構成されるOCP(OpenShift Conatiner Platform)サーバー。
通信フロー
- OCPに外部からアクセスする際には、Host上に構成されているHA Proxyにアクセスする。このHA ProxyがReverse ProxyとなってKVM Guest上のOCPにアクセスする。virbr0は仮想ブリッジではあるが、eth0と同様にHostから認識されるネットワークインターフェースでもあるので（つまりeth1/eth2/eth3とかと同じようなもの）、Host上で実行されるプロセスの１つであるHAProxyからはどちらのネットワークインターフェースも利用できる。HAProxyがOCPにアクセスする際には、virbr0(192.168.122.1)を使ってアクセスする。
- KVM Guestが外部ネットワークにアクセスする時には、そのパケットはNAT(正確にはMasquerade)される。これは、仮想ブリッジでNAT設定をしている際にiptablesがバックグラウンドで構成されるからである。これによりKVM GuestからHost上のある１つのネットワークインターフェース（birbr0）にパケットが届くと、iptablesによってMasqueradeが実施され、そのあとはルーティングに従って適切なネットワークインターフェース（今回の場合はeth0)に従って外部にパケットが放出される。

完成時のHostにおけるネットワーク情報

[root@sno-ocp-vsi ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 02:00:02:66:65:06 brd ff:ff:ff:ff:ff:ff
    altname enp0s3
    altname ens3
    inet 10.240.0.4/24 brd 10.240.0.255 scope global dynamic noprefixroute eth0
       valid_lft 354sec preferred_lft 354sec
    inet6 fe80::2ff:fe66:6506/64 scope link
       valid_lft forever preferred_lft forever
4: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 52:54:00:1a:e4:96 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
5: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master virbr0 state UNKNOWN group default qlen 1000
    link/ether fe:54:00:70:e2:71 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe70:e271/64 scope link
       valid_lft forever preferred_lft forever
6: vnet1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master virbr0 state UNKNOWN group default qlen 1000
    link/ether fe:50:90:0e:45:12 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc50:90ff:fe0e:4512/64 scope link
       valid_lft forever preferred_lft forever

[root@sno-ocp-vsi ~]# ss -anpt
State       Recv-Q      Send-Q           Local Address:Port              Peer Address:Port      Process
LISTEN      0           128               0.0.0.0%eth0:443                    0.0.0.0:*          users:(("haproxy",pid=20754,fd=9))
LISTEN      0           128               0.0.0.0%eth0:9000                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=5))
LISTEN      0           128               0.0.0.0%eth0:6443                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=7))
LISTEN      0           128                  127.0.0.1:5900                   0.0.0.0:*          users:(("qemu-kvm",pid=15273,fd=13))
LISTEN      0           128                  127.0.0.1:5901                   0.0.0.0:*          users:(("qemu-kvm",pid=18382,fd=13))
LISTEN      0           128                    0.0.0.0:111                    0.0.0.0:*          users:(("rpcbind",pid=1037,fd=4),("systemd",pid=1,fd=71))
LISTEN      0           128               0.0.0.0%eth0:80                     0.0.0.0:*          users:(("haproxy",pid=20754,fd=8))
LISTEN      0           32               192.168.122.1:53                     0.0.0.0:*          users:(("dnsmasq",pid=13013,fd=5))
LISTEN      0           128               0.0.0.0%eth0:8022                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=10))
LISTEN      0           128                    0.0.0.0:22                     0.0.0.0:*          users:(("sshd",pid=1254,fd=3))
ESTAB       0           36                  10.240.0.4:22             162.xxx.xxx.xxx:58010      users:(("sshd",pid=23491,fd=4),("sshd",pid=23478,fd=4))
LISTEN      0           128                       [::]:111                       [::]:*          users:(("rpcbind",pid=1037,fd=6),("systemd",pid=1,fd=73))
LISTEN      0           128                       [::]:22                        [::]:*          users:(("sshd",pid=1254,fd=4))


[root@sno-ocp-vsi ~]# virsh net-dumpxml default
<network connections='2'>
  <name>default</name>
  <uuid>90d5a993-a26b-4fe7-9487-406803fba86c</uuid>
  <forward mode='nat'>
    <nat>
      <port start='1024' end='65535'/>
    </nat>
  </forward>
  <bridge name='virbr0' stp='on' delay='0'/>
  <mac address='52:54:00:9d:de:65'/>
  <ip address='192.168.122.1' netmask='255.255.255.0'>
  </ip>
</network>

[root@sno-ocp-vsi ~]# iptables -L -v -n -t nat
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
37455 2304K LIBVIRT_PRT  all  --  *      *       0.0.0.0/0            0.0.0.0/0

Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain LIBVIRT_PRT (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 RETURN     all  --  *      *       192.168.122.0/24     224.0.0.0/24
    0     0 RETURN     all  --  *      *       192.168.122.0/24     255.255.255.255
 4556  273K MASQUERADE  tcp  --  *      *       192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
 1684  144K MASQUERADE  udp  --  *      *       192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
    1    84 MASQUERADE  all  --  *      *       192.168.122.0/24    !192.168.122.0/24

3. 導入の概要

導入には、以下のAnsible playbookを利用する。

Single Node OpenShift (SNO)導入用のplaybook 。Assisted Serviceとして、https://api.openshift.com が呼ばれている。 Assited Installer については以下も参照。
- https://cloud.redhat.com/blog/how-to-use-the-openshift-assisted-installer
- https://github.com/openshift/assisted-service/tree/master/docs/user-guide
Ansible Galaxy内にあるcloudcollection

ただし、以下の注意点がある。

このplaybookでは、既存のVPCを選択することはできない。新規にVPCを作成し、そのVPCにVSIを作成する。
ユーザーが指定したSSH鍵を選択することはできない。(生成された秘密鍵はユーザーに提供される）
Ansible 2.9を利用する必要がある。最初はCentOS7にAnsible 2.9を入れていたが、Jinja2のバージョンが古いために、templateエラーになっており、その原因や回避策を探すのが大変だった（本稿はCentOS9ベースで試している）。
導入にはそこそこ時間を要するので、Ansible実行環境をIBM Cloud上のサーバーに作成し、そこでscreenコマンドなどを利用してセッションが切れないようにすると良い。

2. Software群の導入

2-1. Ansible実行環境の準備。

（自分のMac環境を汚したくない+移動が多いのでインストール作業を止めたくないためscreenとかを使いたいというのもあり）、今回はAnsible実行環境としてCentOS9をIBM Cloud上に準備。

2-2. screenのインストール

CentOS9にscreenコマンドを導入する方法を参照。screenの使い方はここでは説明しない。インストールには時間を要するので途中で離席してもセッションが切れないようにするために、以下はscreenのセッション内で作業する。

2-3. git/jqのインストール

[root@syasuda-centos9 ~]# dnf install -y git jq

（参考）
Single Node OpenShift (SNO)導入用のplaybookの中では、jqコマンドは一見API Keyの自動取得のためだけに使っているように見えるが、実際は、Ansibleの中でlocalhostに対してjqコマンドを実行する場所があるため、jqコマンドは必須である。

2-4. Ansible 2.9のインストール

チュートリアルによると、Ansible 2.9が必要である。CentOS Stream 9にAnsible 2.9の最新版を導入するに従ってAnsibleを導入する。

その上で、必要なAnsible collectionを導入する。

[root@syasuda-centos9 ~]# source env1/bin/activate

(env1) [root@syasuda-centos9 ~]# ansible --version
ansible 2.9.27
  config file = None
  configured module search path = ['/root/.ansible/plugins/modules', '/usr/share/ansible/plugins/modules']
  ansible python module location = /root/env1/lib64/python3.9/site-packages/ansible
  executable location = /root/env1/bin/ansible
  python version = 3.9.17 (main, Aug  9 2023, 00:00:00) [GCC 11.4.1 20230605 (Red Hat 11.4.1-2)]

(env1) [root@syasuda-centos9 ~]# ansible-galaxy collection install ibm.cloudcollection
Process install dependency map
Starting collection install process
Installing 'ibm.cloudcollection:1.49.0' to '/root/.ansible/collections/ansible_collections/ibm/cloudcollection'

(env1) [root@syasuda-centos9 ~]# ansible-galaxy collection install community.libvirt
Process install dependency map
Starting collection install process
Installing 'community.libvirt:1.3.0' to '/root/.ansible/collections/ansible_collections/community/libvirt'

3. セットアップ

3-1. APIキーの設定

何度もコピーするのが面倒なので、APIキーは~/ibmcloud_apikeyというファイルに保管しておき、それを環境変数に呼び出すようにしている。

(env1) [root@syasuda-centos9 ~]# export IC_API_KEY=$(cat ~/ibmcloud_apikey)

このAPIキーで正しくplaybookが呼び出せるかどうかの稼働確認は（チュートリアルに従うと）以下の通り。

(env1) [root@syasuda-centos9 ~]# ansible localhost -m ibm.cloudcollection.ibm_is_images_info -a "name=ibm-debian-11-2-minimal-amd64-1"
[WARNING]: No inventory was parsed, only implicit localhost is available
localhost | SUCCESS => {
    "changed": false,
    "rc": 0,
    "resource": {
        "catalog_managed": null,
        "id": "2023-10-06 01:43:03.855644699 +0000 UTC",
        "images": [
            {
                "access_tags": [],
                "architecture": "amd64",
                "catalog_offering": [
                    {
                        "managed": false,
                        "version": []
                    }
                ],
                "checksum": "38a853cd338d8cea2b01b5b216da8d966310fc6aa3693b6cfb274522ccf6fe3e",
                "crn": "crn:v1:bluemix:public:is:us-south:a/811f8abfbd32425597dc7ba40da98fa6::image:r006-3bfa4c4f-9c9e-454b-83fb-2f473cc015bd",
                "encryption": "none",
                "encryption_key": "",
                "id": "r006-3bfa4c4f-9c9e-454b-83fb-2f473cc015bd",
                "name": "ibm-debian-11-2-minimal-amd64-1",
                "os": "debian-11-amd64",
                "source_volume": "",
                "status": "deprecated",
                "visibility": "public"
            }
        ],
        "name": "ibm-debian-11-2-minimal-amd64-1",
        "resource_group": null,
        "status": null,
        "visibility": null
    },
    "stderr": "",
    "stderr_lines": [],
    "stdout": "data.ibm_is_images.ansible_ibmdebian112minimalamd641: Refreshing state...\n\nWarning: Argument is deprecated\n\nThe generation field is deprecated and will be removed after couple of\nreleases\n\n\nApply complete! Resources: 0 added, 0 changed, 0 destroyed.\n",
    "stdout_lines": [
        "data.ibm_is_images.ansible_ibmdebian112minimalamd641: Refreshing state...",
        "",
        "Warning: Argument is deprecated",
        "",
        "The generation field is deprecated and will be removed after couple of",
        "releases",
        "",
        "",
        "Apply complete! Resources: 0 added, 0 changed, 0 destroyed."
    ]
}

3-2. Single Node OpenShift (SNO)導入用のplaybookの構成

(env1) [root@syasuda-centos9 ~]# git clone https://github.com/IBM/sno-on-ibm-cloud-vpc-ansible.git

(env1) [root@syasuda-centos9 ~]# cd sno-on-ibm-cloud-vpc-ansible/

(env1) [root@syasuda-centos9 sno-on-ibm-cloud-vpc-ansible]# export HOME_DIR=$(pwd)

(env1) [root@syasuda-centos9 sno-on-ibm-cloud-vpc-ansible]# cd ansible

https://console.redhat.com/openshift/downloads にアクセスする。

Pull secretをダウンロード(pull-secret.txt)し、auth/pull-secret.txtに配置。
OpenShift Clsuter Manager API Tokenをダウンロードし、auth/token.txtに配置。

およびOpenShift Cluster Manager API Tokenを

(env1) [root@syasuda-centos9 ansible]# ls -l auth/
total 8
-rw-------. 1 root root 2771 Oct  6 02:37 pull-secret.txt
-rw-------. 1 root root  802 Oct  6 02:37 token.txt

3-3. PlayBook parameterの編集

（デフォルトのテンプレートがあまり役には立たないが）、group_vars/all.exampleをgroup_vars/allにコピー。

(env1) [root@syasuda-centos9 ansible]# cp -p group_vars/all.example group_vars/all

以下のように、group_vars/allを編集する。以下はOCP4.11を利用する例である。

group_vars/all

---
name_prefix: "sno-ocp"
region: "us-south"
zone: "us-south-1"
resource_group_name: "Default"
sno_cluster_name: "sno-cluster1"
sno_domain_name: "test.local"
setup_vsi_gui: false
sno_version: "4.11"

チュートリアルには、4.11までしかサポートしない旨が書かれているが、本稿執筆時点での最新は4.13であり、4.11は使いたくない。よって、以下のように修正すれば4.12や4.13がvalidation checkでエラーになることはない。

roles/validate_parms/tasks/main.yml


（修正前）
- name: Validate OpenShift version
  assert:
    that: "sno_version in ['4.8','4.9','4.10','4.11']"
    fail_msg: "variable sno_version must be one of 4.8, 4.9, 4.10 or 4.11"
  when: sno_version is defined

（修正後）
- name: Validate OpenShift version
  assert:
    that: "sno_version in ['4.8','4.9','4.10','4.11','4.12','4.13']"
    fail_msg: "variable sno_version must be one of 4.8, 4.9, 4.10, 4.11, 4.12 or 4.13"
  when: sno_version is defined

4.11及び4.12では無事インストールできることを確認しました。ただし、4.13では導入中にエラーになりました。原因は不明です。

パラメーターのvalidation check

(env1) [root@syasuda-centos9 ansible]# ansible-playbook validate_parms.yml
[WARNING]: No inventory was parsed, only implicit localhost is available
[WARNING]: provided hosts list is empty, only localhost is available. Note that the implicit localhost does not match 'all'

PLAY [localhost] ******************************************************************************************************************************************************************************************************************

TASK [Gathering Facts] ************************************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] **********************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "IBM Cloud API KEY is defined"
}

TASK [validate_parms : Validate OpenShift version] ********************************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Validate GUI flag] *****************************************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] **************************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Verify that remaining variables are defined and have valid format] *****************************************************************************************************************************************
ok: [localhost] => (item=name_prefix) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "name_prefix",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=region) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "region",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=zone) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "zone",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=resource_group_name) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "resource_group_name",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=sno_cluster_name) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "sno_cluster_name",
    "msg": "All assertions passed"
}

TASK [validate_parms : Look for token file] ***************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that token file exists] *****************************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "Required token file exists"
}

TASK [validate_parms : Look for pull secret file] *********************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ***********************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "Required pull secret file exists"
}

TASK [validate_parms : Save validation status as fact] ****************************************************************************************************************************************************************************
ok: [localhost]

TASK [Success message] ************************************************************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "Parameter validation successful"
}

PLAY RECAP ************************************************************************************************************************************************************************************************************************
localhost                  : ok=12   changed=0    unreachable=0    failed=0    skipped=1    rescued=0    ignored=0

4. 導入: OCP 4.11の場合

(env1) [root@syasuda-centos9 ansible]# ansible-playbook quickstart.yml
[WARNING]: No inventory was parsed, only implicit localhost is available
[WARNING]: provided hosts list is empty, only localhost is available. Note that the implicit localhost does not match 'all'

PLAY [Start install] ***************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [Starting message] ************************************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "Starting end to end install of OpenShift single node cluster on IBM Cloud VPC ..."
}

PLAY [localhost] *******************************************************************************************************************************************************************************************


TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "IBM Cloud API KEY is defined"
}

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "All assertions passed"
}

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
ok: [localhost] => (item=name_prefix) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "name_prefix",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=region) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "region",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=zone) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "zone",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=resource_group_name) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "resource_group_name",
    "msg": "All assertions passed"
}
ok: [localhost] => (item=sno_cluster_name) => {
    "ansible_loop_var": "item",
    "changed": false,
    "item": "sno_cluster_name",
    "msg": "All assertions passed"
}

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "Required token file exists"
}

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "Required pull secret file exists"
}

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Check resource group exists] ****************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Save existing resource group info as fact] **************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Configure VPC] ******************************************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Save VPC as fact] ***************************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Configure VPC Subnet] ***********************************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Save VPC Subnet as fact] ********************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Generate SSH key for localhost if not exists] ***********************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure SSH Key] **************************************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Save SSH Key as fact] ***********************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Retrieve image list] ************************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Set VM image name/id dictionary fact] *******************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Configure VSI] ******************************************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Save VSI as fact] ***************************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Configure Floating IP Address] **************************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Save Floating IP as fact] *******************************************************************************************************************************************************
ok: [localhost]

TASK [provision_kvm_host : Print Floating IP Address] ******************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "IP Address: 52.xxx.xxx.xx"
}

TASK [provision_kvm_host : Configure Security Group Rule to open SSH on the VSI] ***************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open ICMP traffic to the VSI] ******************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open VNC on the VSI] ***************************************************************************************************************************
skipping: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open Haproxy stats] ****************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open HTTPS] ************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open HTTP] *************************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open OpenShift API] ****************************************************************************************************************************
changed: [localhost]

TASK [provision_kvm_host : Configure Security Group Rule to open 8022 for SSH to SNO VM] *******************************************************************************************************************
changed: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
changed: [localhost]

TASK [Wait for VSI to become reachable over SSH] ***********************************************************************************************************************************************************
ok: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [Install Python on remote server for Ansible] *********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx] => (item=yum update -y)
changed: [52.xxx.xxx.xx] => (item=yum install -y python3)

TASK [Collect OS information] ******************************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [Print OS information] ********************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "os_info.stdout_lines": [
        "NAME=\"Rocky Linux\"",
        "VERSION=\"8.8 (Green Obsidian)\"",
        "ID=\"rocky\"",
        "ID_LIKE=\"rhel centos fedora\"",
        "VERSION_ID=\"8.8\"",
        "PLATFORM_ID=\"platform:el8\"",
        "PRETTY_NAME=\"Rocky Linux 8.8 (Green Obsidian)\"",
        "ANSI_COLOR=\"0;32\"",
        "LOGO=\"fedora-logo-icon\"",
        "CPE_NAME=\"cpe:/o:rocky:rocky:8:GA\"",
        "HOME_URL=\"https://rockylinux.org/\"",
        "BUG_REPORT_URL=\"https://bugs.rockylinux.org/\"",
        "SUPPORT_END=\"2029-05-31\"",
        "ROCKY_SUPPORT_PRODUCT=\"Rocky-Linux-8\"",
        "ROCKY_SUPPORT_PRODUCT_VERSION=\"8.8\"",
        "REDHAT_SUPPORT_PRODUCT=\"Rocky Linux\"",
        "REDHAT_SUPPORT_PRODUCT_VERSION=\"8.8\""
    ]
}

TASK [Get Floating IP info from previous play] *************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [format_data_disk : check if extra drive already setup] ***********************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [format_data_disk : Create partition on extra disk] ***************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Create volume group on new partition] *********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Create a logical volume the size of all remaining space in the volume group] ******************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Creates mount directory for extra drive] ******************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Create an xfs filesystem on new lv] ***********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Mount new filesystem] *************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [format_data_disk : Create marker file] ***************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [disable_se_linux : Disable SE Linux on host] *********************************************************************************************************************************************************
[WARNING]: SELinux state change will take effect next reboot
changed: [52.xxx.xxx.xx]

TASK [reboot_vsi : Get VSI data] ***************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Save VSI info as fact] ******************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot VSI] *****************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot message] *************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot task] ****************************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot message] *************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": "Rebooted via Ansible task"
}

TASK [reboot_vsi : Wait for VSI to come back up] ***********************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [reboot_vsi : Success message] ************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": "VSI successfully rebooted"
}

TASK [Show access info] ************************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": [
        "Your VSI has been successfully provisioned !",
        "Login in via SSH using the command:",
        "ssh -i /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/sno-ocp-local-id_rsa root@52.xxx.xxx.xx",
        ""
    ]
}

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [Generate SSH keypair for accessing KVM guests from host] *********************************************************************************************************************************************
changed: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [setup_kvm_host : Install KVM packages on KVM VSI] ****************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Install TPM tools on KVM VSI] *******************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Check if user kvmadmin exists and create if not] ************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Allow KVM tools to be run as kvmadmin user - Step 1] ********************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Allow KVM tools to be run as kvmadmin user - Step 2] ********************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : enable and start libvirtd] **********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Add local ssh key to kvmadmin user] *************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Add user kvmadmin to sudo] **********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Copy new SSH keypair for user kvmadmin Step 1] **************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Copy new SSH keypair for user kvmadmin Step 2] **************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Check for DHCP in default KVM network] **********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Create dir for backup files] ********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Copy template for default network] **************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Disable DHCP in default network config] *********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Stop existing default network] ******************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Undefine existing default network] **************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Redefine default network] ***********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Start default network] **************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Ensure that the default network will be started at boot] ****************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Check if storage pool has been setup] ***********************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Create dir for storage pool] ********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Allow kvmadmin r/w access sorage pool dir] ******************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Give qemu user access to images folder] *********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Set storage pool name] **************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Create pool config from template] ***************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Define storage pool] ****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Start storage pool] *****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_kvm_host : Autostart storage pool] *************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [Get Floating IP info from previous play] *************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [Show success message] ********************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": [
        "KVM has been succesfully setup on your VSI !",
        "You can now run kvm commands by logging in via SSH:",
        "    ssh -i /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/sno-ocp-local-id_rsa root@52.xxx.xxx.xx",
        "For root access you can prefix each command with sudo as shown below:",
        "    sudo dnf install -y my-favorite-utility",
        ""
    ]
}

PLAY [Get VSIs IP and add to inventory if not already there] ***********************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [Get Floating IP info from previous play] *************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [install_gui : Install Server GUI] ********************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [install_gui : Start GUI on boot] *********************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Get VSI data] ***************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Save VSI info as fact] ******************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot VSI] *****************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot message] *************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot task] ****************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Reboot message] *************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Wait for VSI to come back up] ***********************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [reboot_vsi : Success message] ************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [Show access info] ************************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [setup_vnc : Check if VNC setup already] **************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Install VNC server] **********************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Add kvmadmin to the vncserver config] ****************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Create VNC folder for ocpadmin user] *****************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Send session type to VSI] ****************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Upload initial VNC password script] ******************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Initial VNC password script] *************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [setup_vnc : Start VNC service] ***********************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [Get Floating IP info from previous plays] ************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

TASK [Show access info] ************************************************************************************************************************************************************************************
skipping: [52.xxx.xxx.xx]

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [create_utility_vm : Get existing VMs] ****************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [create_utility_vm : Download base image] *************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Uncompress image] ****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Configure the image] *************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Copy base image to libvirt directory] ********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Define utility vm] ***************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Ensure VM is started] ************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Ensure temporary file is deleted] ************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Check if VM configured already] **************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [create_utility_vm : Install ansible] *****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Create setup dir on KVM host] ****************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Copy setup files] ****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Copy playbook] *******************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Run playbook targeting utility vm] ***********************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_utility_vm : Show remote playbook output] *****************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "remote_playbook_out.stdout_lines": [
        "",
        "PLAY [Configure Utility VM] ****************************************************",
        "",
        "TASK [Gathering Facts] *********************************************************",
        "ok: [192.168.122.2]",
        "",
        "TASK [Upgrade all packages] ****************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Install DHCP] ************************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Set DHCP interface] ******************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Create DHCP config] ******************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Start the DHCP service] **************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Install named] ***********************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Create named config] *****************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Create forward dns zone] *************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Create reverse dns zone] *************************************************",
        "changed: [192.168.122.2]",
        "",
        "TASK [Start the named service] *************************************************",
        "changed: [192.168.122.2]",
        "",
        "PLAY RECAP *********************************************************************",
        "192.168.122.2              : ok=11   changed=10   unreachable=0    failed=0    skipped=0    rescued=0    ignored=0   "
    ]
}

TASK [create_utility_vm : Create marker file] **************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [Show VM info] ****************************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": [
        "The VM for installing OpenShift SNO have been created !",
        "",
        "You can access the SNO utility VM from the KVM VSI with:",
        "  ssh root@192.168.122.2",
        ""
    ]
}

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [prepare_sno_install : Assisted installer token] ******************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Parse response] ****************************************************************************************************************************************************************
ok: [localhost]

(env1) [root@syasuda-centos9 ansible]# ansible-playbook quickstart.yml
TASK [prepare_sno_install : Escaped pull secret] ***********************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Get clusters] ******************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Get cluster names] *************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Cluster names  for debugging] **************************************************************************************************************************************************
ok: [localhost] => {
    "cluster_names": []
}

TASK [prepare_sno_install : Clean up any old marker files] *************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Register a cluster] ************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Parse response] ****************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Cluster Id] ********************************************************************************************************************************************************************
ok: [localhost] => {
    "cluster_id": "00ecf36c-0b66-4075-adac-e428df09a671"
}

TASK [prepare_sno_install : Get existing cluster info] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [prepare_sno_install : Existing cluster id] ***********************************************************************************************************************************************************
skipping: [localhost]

TASK [prepare_sno_install : Get infra-envs] ****************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Get infrenv names] *************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Infra-envs names for debugging] ************************************************************************************************************************************************
ok: [localhost] => {
    "infraenv_names": []
}

TASK [prepare_sno_install : Register an infraenv] **********************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Parse response] ****************************************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Infraenv Id] *******************************************************************************************************************************************************************
ok: [localhost] => {
    "infraenv_id": "e2c00400-a30e-4d88-a8a9-ea06b835f841"
}

TASK [prepare_sno_install : Get existing infraenv info] ****************************************************************************************************************************************************
skipping: [localhost]

TASK [prepare_sno_install : Existing infraenv id] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [prepare_sno_install : Get InfraEnv Image Download URL] ***********************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Save download image URL as fact] ***********************************************************************************************************************************************
ok: [localhost]

TASK [prepare_sno_install : Image_url_response.json for  debugging] ****************************************************************************************************************************************
ok: [localhost] => {
    "image_url_response.json": {
        "cluster_id": "00ecf36c-0b66-4075-adac-e428df09a671",
        "cpu_architecture": "x86_64",
        "created_at": "2023-10-06T06:58:05.816218Z",
        "download_url": "https://api.openshift.com/api/assisted-images/images/e2c00400-a30e-4d88-a8a9-ea06b835f841?arch=x86_64&image_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTY1ODk4ODUsInN1
YiI6ImUyYzAwNDAwLWEzMGUtNGQ4OC1hOGE5LWVhMDZiODM1Zjg0MSJ9.Gl_VhBXsIhcnz6C-LIpTN93YwVR7eYZPmG_L0CBawDM&type=full-iso&version=4.10",
        "email_domain": "jp.ibm.com",
        "expires_at": "2023-10-06T10:58:05.000Z",
        "href": "/api/assisted-install/v2/infra-envs/e2c00400-a30e-4d88-a8a9-ea06b835f841",
        "id": "e2c00400-a30e-4d88-a8a9-ea06b835f841",
        "kind": "InfraEnv",
        "name": "sno-cluster1-infraenv",
        "openshift_version": "4.10",
        "org_id": "11320705",
        "proxy": {},
        "pull_secret_set": true,
        "type": "full-iso",
        "updated_at": "2023-10-06T06:58:05.827532Z",
        "user_name": "shiyasu@jp.ibm.com"
    }
}

TASK [prepare_sno_install : Download URL for debugging] ****************************************************************************************************************************************************
ok: [localhost] => {
    "download_url": "https://api.openshift.com/api/assisted-images/images/e2c00400-a30e-4d88-a8a9-ea06b835f841?arch=x86_64&image_token=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjE2OTY1ODk4ODUsInN1YiI6
ImUyYzAwNDAwLWEzMGUtNGQ4OC1hOGE5LWVhMDZiODM1Zjg0MSJ9.Gl_VhBXsIhcnz6C-LIpTN93YwVR7eYZPmG_L0CBawDM&type=full-iso&version=4.10"
}

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [Get ISO download URL from previous play] *************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [create_sno_vm : Get existing VMs] ********************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [create_sno_vm : Get ISO] *****************************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_sno_vm : Generate empty VM disk] **************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_sno_vm : Change VM disk ownership to qemu] ****************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_sno_vm : Define SNO VM] ***********************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_sno_vm : Ensure VM is started] ****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [create_sno_vm : Wait for VM to become reachable over SSH] ********************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [Success message] *************************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": "Your SNO cluster is ready to install"
}

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [start_sno_install : Get  cluster id and infraenv id] *************************************************************************************************************************************************
ok: [localhost]

TASK [start_sno_install : Show cluster_id and infraenv_id for debugging] ***********************************************************************************************************************************
ok: [localhost] => (item={'name': 'Cluster Id', 'value': '00ecf36c-0b66-4075-adac-e428df09a671'}) => {
    "msg": "Cluster Id is 00ecf36c-0b66-4075-adac-e428df09a671"
}
ok: [localhost] => (item={'name': 'Infraenv Id', 'value': 'e2c00400-a30e-4d88-a8a9-ea06b835f841'}) => {
    "msg": "Infraenv Id is e2c00400-a30e-4d88-a8a9-ea06b835f841"
}

TASK [start_sno_install : Check if marker file already exists and install start API call if it does] *******************************************************************************************************
ok: [localhost]

TASK [start_sno_install : Get host status] *****************************************************************************************************************************************************************
FAILED - RETRYING: Get host status (30 retries left).
FAILED - RETRYING: Get host status (29 retries left).
FAILED - RETRYING: Get host status (28 retries left).
FAILED - RETRYING: Get host status (27 retries left).
FAILED - RETRYING: Get host status (26 retries left).
FAILED - RETRYING: Get host status (25 retries left).
FAILED - RETRYING: Get host status (24 retries left).
ok: [localhost]

TASK [start_sno_install : Show host status] ****************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "Host status is Host is ready to be installed"
}

TASK [start_sno_install : Set hostname of single node] *****************************************************************************************************************************************************
ok: [localhost]

TASK [start_sno_install : Call Assisted Installer API] *****************************************************************************************************************************************************
ok: [localhost]

TASK [start_sno_install : Create marker file] **************************************************************************************************************************************************************
changed: [localhost]

TASK [Success message] *************************************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "Your SNO install started successfully !"
}

PLAY [localhost] *******************************************************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [validate_parms : Verify that IBM Cloud API KEY is defined in env if not a var] ***********************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate OpenShift version] *********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Validate GUI flag] ******************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify IC_API_KEY parameter is defined if not in env] *******************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify sno_domain_name parameter] ***************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that remaining variables are defined and have valid format] ******************************************************************************************************************
skipping: [localhost] => (item=name_prefix)
skipping: [localhost] => (item=region)
skipping: [localhost] => (item=zone)
skipping: [localhost] => (item=resource_group_name)
skipping: [localhost] => (item=sno_cluster_name)

TASK [validate_parms : Look for token file] ****************************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that token file exists] ******************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Look for pull secret file] **********************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Verify that pull secret file exists] ************************************************************************************************************************************************
skipping: [localhost]

TASK [validate_parms : Save validation status as fact] *****************************************************************************************************************************************************
skipping: [localhost]

TASK [complete_sno_install : Poll for cluster install completion for 90 minutes] ***************************************************************************************************************************
changed: [localhost]

TASK [complete_sno_install : Wait for polling script to complete] ******************************************************************************************************************************************
FAILED - RETRYING: Wait for polling script to complete (270 retries left).
FAILED - RETRYING: Wait for polling script to complete (269 retries left).
FAILED - RETRYING: Wait for polling script to complete (268 retries left).
FAILED - RETRYING: Wait for polling script to complete (267 retries left).
FAILED - RETRYING: Wait for polling script to complete (266 retries left).
FAILED - RETRYING: Wait for polling script to complete (265 retries left).
FAILED - RETRYING: Wait for polling script to complete (264 retries left).
FAILED - RETRYING: Wait for polling script to complete (263 retries left).
FAILED - RETRYING: Wait for polling script to complete (262 retries left).
FAILED - RETRYING: Wait for polling script to complete (261 retries left).
FAILED - RETRYING: Wait for polling script to complete (260 retries left).
FAILED - RETRYING: Wait for polling script to complete (259 retries left).
FAILED - RETRYING: Wait for polling script to complete (258 retries left).
FAILED - RETRYING: Wait for polling script to complete (257 retries left).
FAILED - RETRYING: Wait for polling script to complete (256 retries left).
FAILED - RETRYING: Wait for polling script to complete (255 retries left).
FAILED - RETRYING: Wait for polling script to complete (254 retries left).
FAILED - RETRYING: Wait for polling script to complete (253 retries left).
FAILED - RETRYING: Wait for polling script to complete (252 retries left).
FAILED - RETRYING: Wait for polling script to complete (251 retries left).
FAILED - RETRYING: Wait for polling script to complete (250 retries left).
FAILED - RETRYING: Wait for polling script to complete (249 retries left).
FAILED - RETRYING: Wait for polling script to complete (248 retries left).
FAILED - RETRYING: Wait for polling script to complete (247 retries left).
FAILED - RETRYING: Wait for polling script to complete (246 retries left).
FAILED - RETRYING: Wait for polling script to complete (245 retries left).
FAILED - RETRYING: Wait for polling script to complete (244 retries left).
FAILED - RETRYING: Wait for polling script to complete (243 retries left).
FAILED - RETRYING: Wait for polling script to complete (242 retries left).
FAILED - RETRYING: Wait for polling script to complete (241 retries left).
FAILED - RETRYING: Wait for polling script to complete (240 retries left).
FAILED - RETRYING: Wait for polling script to complete (239 retries left).
FAILED - RETRYING: Wait for polling script to complete (238 retries left).
FAILED - RETRYING: Wait for polling script to complete (237 retries left).
FAILED - RETRYING: Wait for polling script to complete (236 retries left).
FAILED - RETRYING: Wait for polling script to complete (235 retries left).
FAILED - RETRYING: Wait for polling script to complete (234 retries left).
FAILED - RETRYING: Wait for polling script to complete (233 retries left).
FAILED - RETRYING: Wait for polling script to complete (232 retries left).
FAILED - RETRYING: Wait for polling script to complete (231 retries left).
FAILED - RETRYING: Wait for polling script to complete (230 retries left).
FAILED - RETRYING: Wait for polling script to complete (229 retries left).
FAILED - RETRYING: Wait for polling script to complete (228 retries left).
FAILED - RETRYING: Wait for polling script to complete (227 retries left).
FAILED - RETRYING: Wait for polling script to complete (226 retries left).
FAILED - RETRYING: Wait for polling script to complete (225 retries left).
FAILED - RETRYING: Wait for polling script to complete (224 retries left).
FAILED - RETRYING: Wait for polling script to complete (223 retries left).
FAILED - RETRYING: Wait for polling script to complete (222 retries left).
FAILED - RETRYING: Wait for polling script to complete (221 retries left).
FAILED - RETRYING: Wait for polling script to complete (220 retries left).
FAILED - RETRYING: Wait for polling script to complete (219 retries left).
FAILED - RETRYING: Wait for polling script to complete (218 retries left).
FAILED - RETRYING: Wait for polling script to complete (217 retries left).
FAILED - RETRYING: Wait for polling script to complete (216 retries left).
FAILED - RETRYING: Wait for polling script to complete (215 retries left).
FAILED - RETRYING: Wait for polling script to complete (214 retries left).
FAILED - RETRYING: Wait for polling script to complete (213 retries left).
FAILED - RETRYING: Wait for polling script to complete (212 retries left).
FAILED - RETRYING: Wait for polling script to complete (211 retries left).
FAILED - RETRYING: Wait for polling script to complete (210 retries left).
FAILED - RETRYING: Wait for polling script to complete (209 retries left).
FAILED - RETRYING: Wait for polling script to complete (208 retries left).
FAILED - RETRYING: Wait for polling script to complete (207 retries left).
FAILED - RETRYING: Wait for polling script to complete (206 retries left).
FAILED - RETRYING: Wait for polling script to complete (205 retries left).
FAILED - RETRYING: Wait for polling script to complete (204 retries left).
FAILED - RETRYING: Wait for polling script to complete (203 retries left).
FAILED - RETRYING: Wait for polling script to complete (202 retries left).
FAILED - RETRYING: Wait for polling script to complete (201 retries left).
FAILED - RETRYING: Wait for polling script to complete (200 retries left).
FAILED - RETRYING: Wait for polling script to complete (199 retries left).
FAILED - RETRYING: Wait for polling script to complete (198 retries left).
FAILED - RETRYING: Wait for polling script to complete (197 retries left).
FAILED - RETRYING: Wait for polling script to complete (196 retries left).
FAILED - RETRYING: Wait for polling script to complete (195 retries left).
FAILED - RETRYING: Wait for polling script to complete (194 retries left).
FAILED - RETRYING: Wait for polling script to complete (193 retries left).
FAILED - RETRYING: Wait for polling script to complete (192 retries left).
FAILED - RETRYING: Wait for polling script to complete (191 retries left).
FAILED - RETRYING: Wait for polling script to complete (190 retries left).
FAILED - RETRYING: Wait for polling script to complete (189 retries left).
FAILED - RETRYING: Wait for polling script to complete (188 retries left).
FAILED - RETRYING: Wait for polling script to complete (187 retries left).
FAILED - RETRYING: Wait for polling script to complete (186 retries left).
FAILED - RETRYING: Wait for polling script to complete (185 retries left).
FAILED - RETRYING: Wait for polling script to complete (184 retries left).
FAILED - RETRYING: Wait for polling script to complete (183 retries left).
FAILED - RETRYING: Wait for polling script to complete (182 retries left).
FAILED - RETRYING: Wait for polling script to complete (181 retries left).
FAILED - RETRYING: Wait for polling script to complete (180 retries left).
FAILED - RETRYING: Wait for polling script to complete (179 retries left).
FAILED - RETRYING: Wait for polling script to complete (178 retries left).
FAILED - RETRYING: Wait for polling script to complete (177 retries left).
FAILED - RETRYING: Wait for polling script to complete (176 retries left).
FAILED - RETRYING: Wait for polling script to complete (175 retries left).
FAILED - RETRYING: Wait for polling script to complete (174 retries left).
FAILED - RETRYING: Wait for polling script to complete (173 retries left).
FAILED - RETRYING: Wait for polling script to complete (172 retries left).
FAILED - RETRYING: Wait for polling script to complete (171 retries left).
FAILED - RETRYING: Wait for polling script to complete (170 retries left).
FAILED - RETRYING: Wait for polling script to complete (169 retries left).
FAILED - RETRYING: Wait for polling script to complete (168 retries left).
FAILED - RETRYING: Wait for polling script to complete (167 retries left).
FAILED - RETRYING: Wait for polling script to complete (166 retries left).
FAILED - RETRYING: Wait for polling script to complete (165 retries left).
FAILED - RETRYING: Wait for polling script to complete (164 retries left).
FAILED - RETRYING: Wait for polling script to complete (163 retries left).
FAILED - RETRYING: Wait for polling script to complete (162 retries left).
FAILED - RETRYING: Wait for polling script to complete (161 retries left).
FAILED - RETRYING: Wait for polling script to complete (160 retries left).
FAILED - RETRYING: Wait for polling script to complete (159 retries left).
FAILED - RETRYING: Wait for polling script to complete (158 retries left).
FAILED - RETRYING: Wait for polling script to complete (157 retries left).
FAILED - RETRYING: Wait for polling script to complete (156 retries left).
FAILED - RETRYING: Wait for polling script to complete (155 retries left).
FAILED - RETRYING: Wait for polling script to complete (154 retries left).
FAILED - RETRYING: Wait for polling script to complete (153 retries left).
changed: [localhost]

TASK [complete_sno_install : Shell script output for debugging] ********************************************************************************************************************************************
ok: [localhost] => {
    "poll_for_install_completion": {
        "ansible_job_id": "291884429733.60065",
        "changed": true,
        "failed": false,
        "finished": 0,
        "results_file": "/root/.ansible_async/291884429733.60065",
        "started": 1
    }
}

TASK [complete_sno_install : Shell script output for debugging] ********************************************************************************************************************************************
ok: [localhost] => {
    "job_result": {
        "ansible_job_id": "291884429733.60065",
        "attempts": 119,
        "changed": true,
        "cmd": "scripts/poll-for-install-completion.sh /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/token.txt sno-cluster1\n",
        "delta": "0:39:37.178563",
        "end": "2023-10-06 07:40:23.212414",
        "failed": false,
        "finished": 1,
        "rc": 0,
        "start": "2023-10-06 07:00:46.033851",
        "stderr": "",
        "stderr_lines": [],
        "stdout": "Install successful !",
        "stdout_lines": [
            "Install successful !"
        ]
    }
}

TASK [complete_sno_install : Check result of polling script] ***********************************************************************************************************************************************
ok: [localhost] => {
    "changed": false,
    "msg": "Install completed successfully!"
}

TASK [complete_sno_install : Assisted installer token] *****************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Parse response] ***************************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Get installer status] *********************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Save installer status] ********************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Save SNO ip address] **********************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Get kubeconfig file url] ******************************************************************************************************************************************************
ok: [localhost]

TASK [complete_sno_install : Get kubeconfig file] **********************************************************************************************************************************************************
changed: [localhost]

TASK [complete_sno_install : Get kubeadmin creds] **********************************************************************************************************************************************************
changed: [localhost]

TASK [get_floating_ip_address : Get KVM VSIs floating IP] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Save floating IP as fact] **************************************************************************************************************************************************
skipping: [localhost]

TASK [get_floating_ip_address : Verify that floating IP's ipv4 address is defined] *************************************************************************************************************************
skipping: [localhost]

TASK [Add VSI to Ansible inventory] ************************************************************************************************************************************************************************
skipping: [localhost]

TASK [Generate custom hosts file for SNO cluster] **********************************************************************************************************************************************************
changed: [localhost]

PLAY [kvm_vsi] *********************************************************************************************************************************************************************************************

TASK [setup_haproxy : Install haproxy] *********************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_haproxy : Save original config file] ***********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_haproxy : Copy custom cfg file] ****************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_haproxy : Create stats socker folder] **********************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [setup_haproxy : Restart haproxy] *********************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [Get SNO ip address from previous play] ***************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx]

TASK [Add SNO hostnames to KVM host] ***********************************************************************************************************************************************************************
changed: [52.xxx.xxx.xx]

TASK [Show access info] ************************************************************************************************************************************************************************************
ok: [52.xxx.xxx.xx] => {
    "msg": [
        "The OpenShift SNO cluster is ready ! ",
        "",
        "You can access the KVM host via SSH with:",
        "  ssh -i /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/sno-ocp-local-id_rsa kvmadmin@kvmhost.sno-cluster1.test.local",
        "",
        "Your OpenShift cluster credentials are in the file /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/sno-cluster1-kubeadmin.json",
        "",
        "To access the OpenShift console:",
        "",
        "  1. Add the contents of the file /root/sno-on-ibm-cloud-vpc-ansible/ansible/auth/sno-cluster1.hosts to your /etc/hosts file",
        "",
        "  2. You can then access the OpenShift console at the following  URL:",
        "     https://console-openshift-console.apps.sno-cluster1.test.local",
        "",
        "To access the cluster via the oc CLI run the following command:",
        "",
        "  oc login api.sno-cluster1.test.local:6443 and login with your credentials ",
        ""
    ]
}

PLAY [End to end install of SNO on IBM Cloud VPC complete] *************************************************************************************************************************************************

TASK [Gathering Facts] *************************************************************************************************************************************************************************************
ok: [localhost]

TASK [Ending message] **************************************************************************************************************************************************************************************
ok: [localhost] => {
    "msg": "End to end install of OpenShift single node cluster on IBM Cloud VPC complete !"
}

PLAY RECAP *************************************************************************************************************************************************************************************************
52.xxx.xxx.xx              : ok=82   changed=57   unreachable=0    failed=0    skipped=27   rescued=0    ignored=0
localhost                  : ok=89   changed=21   unreachable=0    failed=0    skipped=98   rescued=0    ignored=0

5. インストール後の稼働確認

5-1. https://console.redhat.com/openshift/での状況

5-2. VPC上での環境確認

該当のVPCが作成されている。
該当のVSIが作成されている。
自動作成されているSecurity Groupはこちら。任意のサイトからアクセスできるようになっているため、通信元を制限することも検討する。

5-3. Ansible実行環境上での出力ファイル確認

(env1) [root@syasuda-centos9 ansible]# ls -l auth/
total 44
-rw-------. 1 root root  2771 Oct  6 02:37 pull-secret.txt
-rw-------. 1 root root   141 Oct  6 07:40 sno-cluster1-kubeadmin.json
-rw-------. 1 root root 12165 Oct  6 07:40 sno-cluster1-kubeconfig
-rw-------. 1 root root   649 Oct  6 07:40 sno-cluster1.hosts
-r--------. 1 root root  3357 Oct  6 06:40 sno-ocp-local-id_rsa
-r--------. 1 root root   726 Oct  6 06:40 sno-ocp-local-id_rsa.pub
-r--------. 1 root root  3357 Oct  6 06:50 sno-ocp-remote-id_rsa
-r--------. 1 root root   726 Oct  6 06:50 sno-ocp-remote-id_rsa.pub
-rw-------. 1 root root   802 Oct  6 02:37 token.txt


(env1) [root@syasuda-centos9 ansible]# cat auth/sno-cluster1.hosts
52.xxx.xxx.xxx	api.sno-cluster1.test.local
52.xxx.xxx.xxx	oauth-openshift.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	console-openshift-console.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	grafana-openshift-monitoring.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	thanos-querier-openshift-monitoring.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	prometheus-k8s-openshift-monitoring.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	alertmanager-main-openshift-monitoring.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	prometheus-k8s-openshift-monitoring.apps.sno-cluster1.test.local
52.xxx.xxx.xxx	kvmhost.sno-cluster1.test.local
52.xxx.xxx.xxx	sno.sno-cluster1.test.local

5-4. /etc/hostsへのホスト情報の追加

アクセス元端末（今回はAnsibleの実行環境や、自分のMac端末など）に、/etc/hostsに、上記で生成されていたhostsの情報を追加する。

(env1) [root@syasuda-centos9 ansible]# cat auth/sno-cluster1.hosts >> /etc/hosts

5-5. Web/CLIでのコンソールアクセス

パスワードは以下に記述されている。

sno-cluster1-kubeadmin.json

(env1) [root@syasuda-centos9 ansible]# cat auth/sno-cluster1-kubeadmin.json
{"console_url":"https://console-openshift-console.apps.sno-cluster1.test.local","password":"xxxxx-xxxxx-xxxxx-xxxxx","username":"kubeadmin"}

Webアクセスは、今回の場合は https://console-openshift-console.apps.sno-cluster1.test.local にアクセスする。

CLIアクセスは以下の通り。

(env1) [root@syasuda-centos9 ansible]# oc login api.sno-cluster1.test.local:6443 --insecure-skip-tls-verify=true -u kubeadmin -p xxxxx-xxxxx-xxxxx-xxxxx
WARNING: Using insecure TLS client config. Setting this option is not supported!

Login successful.

You have access to 68 projects, the list has been suppressed. You can list all projects with 'oc projects'

Using project "default".


(env1) [root@syasuda-centos9 ansible]# oc get clusterversion
NAME      VERSION   AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.11.49   True        False         14h     Cluster version is 4.11.49

(env1) [root@syasuda-centos9 ansible]# oc get nodes
NAME                          STATUS   ROLES           AGE   VERSION
sno.sno-cluster1.test.local   Ready    master,worker   14h   v1.24.16+7aa7ea9

(env1) [root@syasuda-centos9 ansible]# oc debug node/sno.sno-cluster1.test.local
Temporary namespace openshift-debug-8r5f5 is created for debugging node...
Starting pod/snosno-cluster1testlocal-debug ...
To use host binaries, run `chroot /host`
Pod IP: 192.168.122.3
If you don't see a command prompt, try pressing enter.
sh-4.4# chroot /host
sh-4.4# cat /etc/redhat-release
Red Hat Enterprise Linux CoreOS release 4.11
sh-4.4# uname -a
Linux sno.sno-cluster1.test.local 4.18.0-372.71.1.el8_6.x86_64 #1 SMP Fri Aug 25 08:34:17 EDT 2023 x86_64 x86_64 x86_64 GNU/Linux

5-6. KVM-VSIへの接続

(env1) [root@syasuda-centos9 ansible]# ssh -i auth/sno-ocp-local-id_rsa root@kvmhost.sno-cluster1.test.local

6. KVM環境詳細

[root@sno-ocp-vsi ~]# cat /etc/redhat-release
Rocky Linux release 8.8 (Green Obsidian)

[root@sno-ocp-vsi ~]# uname -a
Linux sno-ocp-vsi 4.18.0-477.27.1.el8_8.x86_64 #1 SMP Wed Sep 20 15:55:39 UTC 2023 x86_64 x86_64 x86_64 GNU/Linux

[root@sno-ocp-vsi ~]# cat /etc/hosts
127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
# BEGIN ANSIBLE MANAGED BLOCK
192.168.122.3   api.sno-cluster1.test.local
192.168.122.3   oauth-openshift.apps.sno-cluster1.test.local
192.168.122.3   console-openshift-console.apps.sno-cluster1.test.local
192.168.122.3   grafana-openshift-monitoring.apps.sno-cluster1.test.local
192.168.122.3   thanos-querier-openshift-monitoring.apps.sno-cluster1.test.local
192.168.122.3   prometheus-k8s-openshift-monitoring.apps.sno-cluster1.test.local
192.168.122.3   alertmanager-main-openshift-monitoring.apps.sno-cluster1.test.local
# END ANSIBLE MANAGED BLOCK

[root@sno-ocp-vsi ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host
       valid_lft forever preferred_lft forever
2: eth0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc mq state UP group default qlen 1000
    link/ether 02:00:02:66:65:06 brd ff:ff:ff:ff:ff:ff
    altname enp0s3
    altname ens3
    inet 10.240.0.4/24 brd 10.240.0.255 scope global dynamic noprefixroute eth0
       valid_lft 354sec preferred_lft 354sec
    inet6 fe80::2ff:fe66:6506/64 scope link
       valid_lft forever preferred_lft forever
4: virbr0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
    link/ether 52:54:00:1a:e4:96 brd ff:ff:ff:ff:ff:ff
    inet 192.168.122.1/24 brd 192.168.122.255 scope global virbr0
       valid_lft forever preferred_lft forever
5: vnet0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master virbr0 state UNKNOWN group default qlen 1000
    link/ether fe:54:00:70:e2:71 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc54:ff:fe70:e271/64 scope link
       valid_lft forever preferred_lft forever
6: vnet1: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue master virbr0 state UNKNOWN group default qlen 1000
    link/ether fe:50:90:0e:45:12 brd ff:ff:ff:ff:ff:ff
    inet6 fe80::fc50:90ff:fe0e:4512/64 scope link
       valid_lft forever preferred_lft forever

[root@sno-ocp-vsi ~]# ss -anpt
State       Recv-Q      Send-Q           Local Address:Port              Peer Address:Port      Process
LISTEN      0           128               0.0.0.0%eth0:443                    0.0.0.0:*          users:(("haproxy",pid=20754,fd=9))
LISTEN      0           128               0.0.0.0%eth0:9000                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=5))
LISTEN      0           128               0.0.0.0%eth0:6443                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=7))
LISTEN      0           128                  127.0.0.1:5900                   0.0.0.0:*          users:(("qemu-kvm",pid=15273,fd=13))
LISTEN      0           128                  127.0.0.1:5901                   0.0.0.0:*          users:(("qemu-kvm",pid=18382,fd=13))
LISTEN      0           128                    0.0.0.0:111                    0.0.0.0:*          users:(("rpcbind",pid=1037,fd=4),("systemd",pid=1,fd=71))
LISTEN      0           128               0.0.0.0%eth0:80                     0.0.0.0:*          users:(("haproxy",pid=20754,fd=8))
LISTEN      0           32               192.168.122.1:53                     0.0.0.0:*          users:(("dnsmasq",pid=13013,fd=5))
LISTEN      0           128               0.0.0.0%eth0:8022                   0.0.0.0:*          users:(("haproxy",pid=20754,fd=10))
LISTEN      0           128                    0.0.0.0:22                     0.0.0.0:*          users:(("sshd",pid=1254,fd=3))
ESTAB       0           36                  10.240.0.4:22             162.xxx.xxx.xxx:58010      users:(("sshd",pid=23491,fd=4),("sshd",pid=23478,fd=4))
LISTEN      0           128                       [::]:111                       [::]:*          users:(("rpcbind",pid=1037,fd=6),("systemd",pid=1,fd=73))
LISTEN      0           128                       [::]:22                        [::]:*          users:(("sshd",pid=1254,fd=4))


[root@sno-ocp-vsi ~]# iptables -L -v -n -t nat
Chain PREROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain INPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain POSTROUTING (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination
37455 2304K LIBVIRT_PRT  all  --  *      *       0.0.0.0/0            0.0.0.0/0

Chain OUTPUT (policy ACCEPT 0 packets, 0 bytes)
 pkts bytes target     prot opt in     out     source               destination

Chain LIBVIRT_PRT (1 references)
 pkts bytes target     prot opt in     out     source               destination
    0     0 RETURN     all  --  *      *       192.168.122.0/24     224.0.0.0/24
    0     0 RETURN     all  --  *      *       192.168.122.0/24     255.255.255.255
 4556  273K MASQUERADE  tcp  --  *      *       192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
 1684  144K MASQUERADE  udp  --  *      *       192.168.122.0/24    !192.168.122.0/24     masq ports: 1024-65535
    1    84 MASQUERADE  all  --  *      *       192.168.122.0/24    !192.168.122.0/24

HAproxyが稼働している

[root@sno-ocp-vsi ~]# ps -ef|grep haproxy
root       20861       1  0 07:41 ?        00:00:00 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -f /etc/haproxy/conf.d -p /run/haproxy.pid
haproxy    20863   20861  0 07:41 ?        00:00:15 /usr/sbin/haproxy -Ws -f /etc/haproxy/haproxy.cfg -f /etc/haproxy/conf.d -p /run/haproxy.pid
root       40952   40923  0 22:00 pts/0    00:00:00 grep --color=auto haproxy

KVM環境ではutility用VMとSNO用VMが稼働している

[root@sno-ocp-vsi ~]# virsh list --all
 Id   Name              State
---------------------------------
 2    sno_utility_vm    running
 3    sno-cluster1-vm   running

[root@sno-ocp-vsi ~]# virsh domblklist sno_utility_vm --details
 Type   Device   Target   Source
---------------------------------------------------------------------------------------
 file   disk     vda      /mnt/kvmimgs/images/sno-kvm-utilities-fedora34.x86_64.qcow2

[root@sno-ocp-vsi ~]# virsh domblklist sno-cluster1-vm --details
 Type   Device   Target   Source
---------------------------------------------------------------------
 file   disk     vda      /mnt/kvmimgs/images/sno-cluster1-vm.qcow2
 file   cdrom    sda      /tmp/discovery_sno-cluster1.iso

[root@sno-ocp-vsi ~]# ls -l /tmp/discovery_sno-cluster1.iso
-rw-r--r-- 1 qemu qemu 1051721728 Oct  6 06:58 /tmp/discovery_sno-cluster1.iso

network

[root@sno-ocp-vsi ~]# virsh net-list --all
 Name      State    Autostart   Persistent
--------------------------------------------
 default   active   yes         yes

[root@sno-ocp-vsi ~]# cat /usr/share/libvirt/networks/default.xml
<network>
  <name>default</name>
  <bridge name='virbr0'/>
  <forward/>
  <ip address='192.168.122.1' netmask='255.255.255.0'>
    <dhcp>
      <range start='192.168.122.2' end='192.168.122.254'/>
    </dhcp>
  </ip>
</network>

[root@sno-ocp-vsi ~]# virsh net-info default
Name:           default
UUID:           90d5a993-a26b-4fe7-9487-406803fba86c
Active:         yes
Persistent:     yes
Autostart:      yes
Bridge:         virbr0

forward modeがnat

[root@sno-ocp-vsi ~]# virsh net-dumpxml default
<network connections='2'>
  <name>default</name>
  <uuid>90d5a993-a26b-4fe7-9487-406803fba86c</uuid>
  <forward mode='nat'>
    <nat>
      <port start='1024' end='65535'/>
    </nat>
  </forward>
  <bridge name='virbr0' stp='on' delay='0'/>
  <mac address='52:54:00:9d:de:65'/>
  <ip address='192.168.122.1' netmask='255.255.255.0'>
  </ip>
</network>

KVMではDHCPは構成されていない

[root@sno-ocp-vsi ~]# virsh pool-list --all
 Name      State    Autostart
-------------------------------
 default   active   yes

[root@sno-ocp-vsi ~]# virsh net-dhcp-leases default
 Expiry Time   MAC address   Protocol   IP address   Hostname   Client ID or DUID
-----------------------------------------------------------------------------------

7. Utility Server環境詳細

KVMサーバーに、SSH鍵をコピー

(env1) [root@syasuda-centos9 ansible]# scp -i auth/sno-ocp-local-id_rsa auth/sno-ocp-remote-id_rsa root@kvmhost.sno-cluster1.test.local:/root/.

Utility Serverへのログイン

(env1) [root@syasuda-centos9 ansible]# ssh -i auth/sno-ocp-local-id_rsa root@kvmhost.sno-cluster1.test.local

[root@sno-ocp-vsi ~]# ssh -i sno-ocp-remote-id_rsa root@192.168.122.2

[root@utilities ~]# cat /etc/redhat-release
Fedora release 34 (Thirty Four)

[root@utilities ~]# uname -a
Linux utilities.test.local 5.11.12-300.fc34.x86_64 #1 SMP Wed Apr 7 16:31:13 UTC 2021 x86_64 x86_64 x86_64 GNU/Linux

dhcp

[root@utilities ~]# ps -ef|grep dhcp
dhcpd      15985       1  0 06:56 ?        00:00:00 /usr/sbin/dhcpd -f -cf /etc/dhcp/dhcpd.conf -user dhcpd -group dhcpd --no-pid eth0
root       17517   17469  0 11:33 pts/0    00:00:00 grep --color=auto dhcp

[root@utilities ~]# cat /etc/dhcp/dhcpd.conf
# Static DHCP

subnet 192.168.122.0 netmask 255.255.255.0 {
 option routers 192.168.122.1;
 option domain-name-servers 192.168.122.2;
 option domain-name "test.local";
 use-host-decl-names on;

 # Static entries
 host sno { hardware ethernet  54:50:90:0e:45:12;  fixed-address 192.168.122.3; option host-name "sno.sno-cluster1.uste"; }


 # this will not give out addresses to hosts not listed above
 deny unknown-clients;
}

namedの構成。名前解決は8.8.8.8で行っている。

[root@utilities ~]# ps -ef | grep named
named      17170       1  0 06:56 ?        00:00:06 /usr/sbin/named -u named -c /etc/named.conf
root       17520   17469  0 11:33 pts/0    00:00:00 grep --color=auto named

[root@utilities ~]# cat /etc/named.conf
//
// named.conf
//
// Provided by Red Hat bind package to configure the ISC BIND named(8) DNS
// server as a caching only nameserver (as a localhost DNS resolver only).
//
// See /usr/share/doc/bind*/sample/ for example named configuration files.
//
// See the BIND Administrator's Reference Manual (ARM) for details about the
// configuration located in /usr/share/doc/bind-{version}/Bv9ARM.html


options {
        listen-on port 53 { 127.0.0.1; 192.168.122.0/24; };
        filter-aaaa-on-v4 yes;
        // listen-on-v6 port 53 { ::1; };
        directory       "/var/named";
        dump-file       "/var/named/data/cache_dump.db";
        statistics-file "/var/named/data/named_stats.txt";
        memstatistics-file "/var/named/data/named_mem_stats.txt";
        recursing-file  "/var/named/data/named.recursing";
        secroots-file   "/var/named/data/named.secroots";
        allow-query     { localhost; };

        /*
         - If you are building an AUTHORITATIVE DNS server, do NOT enable recursion.
         - If you are building a RECURSIVE (caching) DNS server, you need to enable
           recursion.
         - If your recursive DNS server has a public IP address, you MUST enable access
           control to limit queries to your legitimate users. Failing to do so will
           cause your server to become part of large scale DNS amplification
           attacks. Implementing BCP38 within your network would greatly
           reduce such attack surface
        */
        recursion yes;
        allow-recursion { 127.0.0.1; 192.168.122.0/24; };

        dnssec-enable yes;
        dnssec-validation yes;

        /* Path to ISC DLV key */
        bindkeys-file "/etc/named.root.key";

        managed-keys-directory "/var/named/dynamic";

        pid-file "/run/named/named.pid";
        session-keyfile "/run/named/session.key";

        # Setup Google's dns as forwarders
        forwarders {
              8.8.8.8;
        };

};

logging {
        channel default_debug {
                file "data/named.run";
                severity dynamic;
        };
};

zone "." IN {
        type hint;
        file "named.ca";
};

include "/etc/named.rfc1912.zones";
include "/etc/named.root.key";

zone "test.local" {
   type master;
   file "/var/named/dynamic/forward.db";
   allow-update { none; };
   allow-query { any; };
};

zone "122.168.192.in-addr.arpa" IN {
      type master;
      file "/var/named/dynamic/192.168.122.db";
      allow-update { none; };
      allow-query { any; };
};

8. アンインストール

/etc/hostsから削除
削除用のplaybookを実行

(env1) [root@syasuda-centos9 ansible]# ansible-playbook teardown_kvm_host.yml

クラスターのアーカイブ

9. OCP 4.11以外のバージョンでのインストールについて

OCP 4.12の場合（成功）

group_vars/all

---
name_prefix: "sno-ocp"
region: "us-south"
zone: "us-south-1"
resource_group_name: "Default"
sno_cluster_name: "sno-cluster1"
sno_domain_name: "test.local"
setup_vsi_gui: false
sno_version: "4.12"

(env1) [root@syasuda-centos9 ansible]# oc get clusterversion
NAME      VERSION   AVAILABLE   PROGRESSING   SINCE   STATUS
version   4.12.35   True        False         4h23m   Cluster version is 4.12.35

OCP 4.13の場合（失敗）

何度やってもここでいつも失敗する。理由は不明。。。

group_vars/all

---
name_prefix: "sno-ocp"
region: "us-south"
zone: "us-south-1"
resource_group_name: "Default"
sno_cluster_name: "sno-cluster1"
sno_domain_name: "test.local"
setup_vsi_gui: false
sno_version: "4.13"