TP Mirroring Ceph S3 ANF CEPH 2022 Sébastien Geiger [ceph: root@cna1 /]# radosgw-admin realm create --rgw-realm=demodom --default [ceph: root@cna1 /]# #radosgw-admin zonegroup delete --rgw-zonegroup=default [ceph: root@cna1 /]# radosgw-admin zonegroup create --rgw-zonegroup=euzone --endpoints=http://cna4:80 --master --default [ceph: root@cna1 /]# radosgw-admin zone create --rgw-zonegroup=euzone --rgw-zone=fr-east-1 --endpoints=http://cna4:80 [ceph: root@cna1 /]# radosgw-admin user create --uid=zone.user --display-name="Zone user" --system --access-key="s3frKeyAcc" --secret="s3frSecret" [ceph: root@cna1 /]# radosgw-admin zone modify --rgw-zone=fr-east-1 --access-key=s3frKeyAcc --secret=s3frSecret [ceph: root@cna1 /]# radosgw-admin period update --commit [ceph: root@cna1 /]# ceph orch apply rgw euzone --realm=demodom --zone=fr-east-1 --placement=cna4 [ceph: root@cna1 /]# radosgw-admin user create --uid=johndoe --display-name="John Doe" --access-key="s3johnKeyAcc" --secret="s3johnKeyMagaSecret" # vérification [ceph: root@cna1 /]# curl http://cna4:80 |xmllint --format - depuis cephclt [almalinux@cephclt ~]$ sudo dnf install -y epel-release [almalinux@cephclt ~]$ sudo dnf install -y awscli [almalinux@cephclt ~]$ vi s3cfg.txt #fichier source s3cfg.txt export AWS_ACCESS_KEY_ID=s3johnKeyAcc export AWS_SECRET_ACCESS_KEY=s3johnKeyMagaSecret export AWS_DEFAULT_REGION=fr-east-1 alias awsa='aws --endpoint-url http://cna4' alias awsb='aws --endpoint-url http://cnb4' export PYTHONWARNINGS="ignore" [almalinux@cephclt ~]$ source s3cfg.txt [almalinux@cephclt ~]$ awsa s3api create-bucket --bucket mybucket [almalinux@cephclt ~]$ awsa s3 ls 2022-08-26 18:32:47 mybucket [almalinux@cephclt ~]$ awsa s3 cp /etc/hosts s3://mybucket/hostcna4 upload: ../../etc/hosts to s3://mybucket/hostcna4 [almalinux@cephclt ~]$ awsa s3 ls s3://mybucket 2022-08-26 18:33:27 458 hostcna4 #ajout deuxieme zone [ceph: root@cnb1 /]# radosgw-admin realm pull --url=http://cna4:80 --access-key=s3frKeyAcc --secret-key=s3frSecret ... "id": "6381a917-ce05-4f09-8cb9-fe98fc01eda3", "name": "demodom", ... [ceph: root@cnb1 /]# radosgw-admin zone create --rgw-zonegroup=euzone --rgw-zone=fr-west-1 --endpoints=http://cnb4:80 --access-key=s3frKeyAcc --secret-key=s3frSecret [ceph: root@cnb1 /]# radosgw-admin period update --commit [ceph: root@cnb1 /]# ceph orch apply rgw euzone --realm=demodom --zone=fr-west-1 --placement=cnb4 Scheduled rgw.euzone update... [ceph: root@cnb1 /]# radosgw-admin sync status realm 6381a917-ce05-4f09-8cb9-fe98fc01eda3 (demodom) zonegroup e1026500-1d22-4d09-85e2-68fa2e0b99cc (euzone) zone 27369ecd-dd40-4ad6-8483-5465d87b800a (fr-west-1) metadata sync syncing full sync: 0/64 shards incremental sync: 64/64 shards metadata is caught up with master data sync source: f9026b18-ad0d-468c-983f-5a5df76df7be (fr-east-1) syncing full sync: 0/128 shards incremental sync: 128/128 shards data is caught up with source # vérification [almalinux@cephclt ~]$ curl http://cnb4:80 |xmllint --format - [almalinux@cephclt ~]$ awsb s3 ls 2022-08-26 18:32:47 mybucket [almalinux@cephclt ~]$ awsb s3 ls s3://mybucket 2022-08-26 18:33:27 458 hostcna4 # depuis la deuxieme zone on peux creer des nouveaux buckets [almalinux@cephclt ~]$ awsb s3api create-bucket --bucket bkcnb [almalinux@cephclt ~]$ awsb s3 cp /etc/hosts s3://bkcnb/hostcnb4 upload: ../../etc/hosts to s3://bkcnb/hostcnb4 # les buckets, les data et les meta data sont synchronisés [almalinux@cephclt ~]$ awsa s3 ls s3://bkcnb 2022-08-26 18:54:40 458 hostcnb4 # vérifier les pools du cluster cna et cnb avec le commande ceph df # chaque cluster à des pools des noms de ses zones ;) # arrêt des rgw qui gère la master zone [ceph: root@cna1 /]# ceph orch ps |grep euzone rgw.euzone.cna4.uptgcn cna4 *:80 running (24s) 16s ago 35m 101M - 17.2.3 0912465dcea5 2c222082e41a [ceph: root@cna1 /]# ceph orch daemon stop rgw.euzone.cna4.uptgcn Scheduled to stop rgw.euzone.cna4.uptgcn on host 'cna4' [ceph: root@cna1 /]# ceph orch ps |grep euzone rgw.euzone.cna4.uptgcn cna4 *:80 stopped 1s ago 36m - - [almalinux@cephclt ~]$ awsb s3 cp /etc/hosts s3://bkcnb/hostcnb4v2 upload: ../../etc/hosts to s3://bkcnb/hostcnb4v2 [almalinux@cephclt ~]$ awsb s3 ls s3://bkcnb 2022-08-26 18:54:40 458 hostcnb4 2022-08-26 19:08:26 458 hostcnb4v2 # remarque: on peux toujours ajouter des fichiers dans les buckets éxistant [almalinux@cephclt ~]$ awsb s3api create-bucket --bucket bkcnbv2 An error occurred (UnknownError) when calling the CreateBucket operation (reached max retries: 4): Unknown # il n'est pas possible de creer de nouveaux bucket si ma master zone n'est pas disponible # qui est master ? [ceph: root@cnb1 /]# radosgw-admin zonegroup get ... "hostnames_s3website": [], "master_zone": "f9026b18-ad0d-468c-983f-5a5df76df7be", "zones": [ "id": "27369ecd-dd40-4ad6-8483-5465d87b800a", "name": "fr-west-1", "endpoints": [ "http://cnb4:80" ], ... "id": "f9026b18-ad0d-468c-983f-5a5df76df7be", "name": "fr-east-1", "endpoints": [ "http://cna4:80" ], ... # promote de cnb4 [ceph: root@cnb1 /]# radosgw-admin zone modify --rgw-zone=fr-west-1 --master [ceph: root@cnb1 /]# radosgw-admin zonegroup modify --rgw-zonegroup=euzone --master [ceph: root@cnb1 /]# radosgw-admin period update --commit ... "hostnames_s3website": [], "master_zone": "27369ecd-dd40-4ad6-8483-5465d87b800a", "zones": [ ... "realm_name": "demodom", "realm_epoch": 3 ... # remarque : master_zone a été modifié avec l'id de la zone du clusterB et realm_epoch a été augmenté d'une période, c'est donc nouvelle configuration. [almalinux@cephclt ~]$ awsb s3api create-bucket --bucket bkcnbv2 [almalinux@cephclt ~]$ awsb s3 cp /etc/hosts s3://bkcnb/hostcnb4v2 upload: ../../etc/hosts to s3://bkcnb/hostcnb4v2 #remarque il est possible de creer des nouveaux buckets et de rajouter des fichiers. # le promote de cnb4 est validé. [ceph: root@cna1 /]# ceph orch daemon start rgw.euzone.cna4.uptgcn Scheduled to start rgw.euzone.cna4.uptgcn on host 'cna4' [ceph: root@cna1 /]# ceph orch ps |grep euzone rgw.euzone.cna4.uptgcn cna4 *:80 running (4s) 1s ago 68m 102M - 17.2.3 0912465dcea5 1cdc70cedf1e [almalinux@cephclt ~]$ awsb s3 ls 2022-08-26 18:53:38 bkcnb 2022-08-26 19:28:59 bkcnbv2 2022-08-26 18:32:47 mybucket [almalinux@cephclt ~]$ awsa s3 ls 2022-08-26 18:53:38 bkcnb 2022-08-26 19:28:59 bkcnbv2 2022-08-26 18:32:47 mybucket # remarque: après le redémarrage de rgw.euzone.cna4 les buckets et les data sont resynchronisés. [ceph: root@cnb1 /]# radosgw-admin sync status realm 6381a917-ce05-4f09-8cb9-fe98fc01eda3 (demodom) zonegroup e1026500-1d22-4d09-85e2-68fa2e0b99cc (euzone) zone 27369ecd-dd40-4ad6-8483-5465d87b800a (fr-west-1) metadata sync no sync (zone is master) data sync source: f9026b18-ad0d-468c-983f-5a5df76df7be (fr-east-1) # Remarque : la zone sur le clusterB est bien marster [ceph: root@cna1 /]# radosgw-admin sync status realm 6381a917-ce05-4f09-8cb9-fe98fc01eda3 (demodom) zonegroup e1026500-1d22-4d09-85e2-68fa2e0b99cc (euzone) zone f9026b18-ad0d-468c-983f-5a5df76df7be (fr-east-1) metadata sync syncing full sync: 0/64 shards incremental sync: 64/64 shards metadata is caught up with master data sync source: 27369ecd-dd40-4ad6-8483-5465d87b800a (fr-west-1) syncing full sync: 0/128 shards incremental sync: 128/128 shards data is caught up with source # Remarque : fr-east-1 se synchronise depuis fr-west-1