Resumer des commandes ceph


# status du cluster
ceph -s
# watch message
ceph -w
# detail des messages
ceph health detail

# liste les osd
ceph osd tree
# liste seulement les osd down
ceph osd tree down

# recherche un osd
ceph osd find id
ceph device ls
ceph device ls-by-host <host>
ceph device info <devid>
device light on|off <devid> 

# disque usage
ceph df
# disque usage par nodes et class
ceph osd df tree hdd
# paramestres des pools
ceph osd pool ls detail
ceph osd pool get <poolname> all

# poid d'un OSD
ceph osd reweight-by-utilization [<max_change:float>] [<max_osds:int>]
ceph osd test-reweight-by-utilization [<max_change:float>] [<max_osds:int>]
# réajuste le poids d'un ods 
ceph osd reweight {id} {wght}

# liste des osd qui un poids different de 1
ceph osd tree -f json-pretty | jq '.nodes[]|select(.type=="osd")|select(.reweight != 1)|.id'
3
34


# crush rule
ceph osd crush rule dump
ceph osd crush rule ls

# version des services
ceph versions

# ceph crash service
ceph crash ls
ceph crash info <id>
ceph crash rm <id>
ceph crash archive <id>
ceph crash archive-all

# avant un reboot d'un node
ceph ods set noout

#autre flags 
ceph osd set nobackfill
ceph osd set norebalance
ceph osd set norecover
ceph osd set noscrub
ceph osd set nodeep-scrub    
# remarque: utiliser ceph osd unset pour les désactiver.


# infos des pgs
ceph pg dump pgs_brief
ceph pg dump pools
ceph pg dump osds
ceph pg ls-by-pool <poolname>

# vérification des pg
ceph pg scrub {pg-id}
ceph pg deep-scrub {pg-id}

# réparation pg inconsistent
ceph pg repair {pg-id}
voir How to resolve "1 scrub errors/Possible data damage: 1 pg inconsistent"
https://www.suse.com/support/kb/doc/?id=000019694

# autoscale pg
ceph osd pool autoscale-status
ceph osd pool set mypool target_size_bytes 100T

# optimisation du placement
ceph balancer status
 

# parametres activés 
ceph config dump
# exemple 
ceph config set osd osd_scrub_auto_repair true
ceph config set osd osd_scrub_during_recovery true  
 
 
ceph osd dump |grep ratio
ceph osd set-full-ratio 0.95
ceph osd set-nearfull-ratio 0.85
ceph osd set-backfillfull-ratio 0.9

tuned-adm list
tuned-adm active
tuned-adm profile network-latency
tuned-adm profile network-throughput


#orch

ceph orch ls
ceph orch ps
ceph orch orch daemon start|stop|restart|reconfig <name>
ceph orch host label add <host> _admin
ceph orch upgrade start --image quay.io/ceph/ceph:v17.2.1


# rbd bench performance
rados bench -p prbd 10 write --no-cleanup -n client.prbd
rados bench -p prbd 10 seq  -n client.prbd
rados bench -p prbd 10 rand  -n client.prbd
rados cleanup -p prbd -n client.prbd

https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/1.3/html/administration_guide/benchmarking_performance

# iostat depuis le cluster
rbd perf image iotop -p prbd
rbd perf image iostat -p prbd

rbd perf image iotop --pool volume
rbd perf image iostat --pool volume


#depuis le client
fio --filename=/mnt/rbd/fio.txt --size=20G --bs=4M --numjobs=1 --rw=write --name=fio --direct=1 --runtime=30 --iodepth=32  --ioengine=libaio
fio --filename=/mnt/rbd/fio.txt --size=20G --bs=4M --numjobs=1 --rw=read --name=fio --direct=1 --runtime=30 --iodepth=32  --ioengine=libaio



#cephfs voir les opérations
ceph fs dump
cephadm enter --name mds.moncfs.ceph2.nuaeir
ceph daemonperf mds.moncfs.ceph2.nuaeir
ceph daemon mds.cephfs.<host> dump_ops_in_flight
ceph daemon mds.moncfs.ceph2.nuaeir session ls


les commandes de la documentation
https://docs.ceph.com/en/latest/rados/operations/control/


# Remplacement d'un osd
## reperer le disque
ceph-volume lvm list
noter le bon devices =>/dev/sdb

## sort l'osd du cluster et attendre que l'osd soit pret a être detruit.
ceph osd out {id}
while ! ceph osd safe-to-destroy osd.{id} ; do sleep 10 ; done

## Détruire l'osd
sudo systemctl stop ceph-fsid@osd.{id}.service
ceph osd destroy {id} --yes-i-really-mean-it

## Changer le matériel puis préparer le nouveaux disque
ceph-volume lvm zap /dev/sdX --force
ceph-volume lvm create --osd-id {id} --data /dev/sdX

## Redémarrer l'osd
systemctl start ceph-fsid@osd.{id}.service


# Destruction d'un osd
ceph osd purge id --yes-i-really-mean-it
puis effacer les information sur le disque
ceph-volume inventory
ceph-volume lvm zap /dev/sdb --destroy


# quincy autotune_memory
ceph config set osd osd_memory_target_autotune true
ceph config set mgr mgr/cephadm/autotune_memory_target_ratio 0.7 # 70% de la ram système, attention si vous faite de l'hyperconvegence.
# voir https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/5/html-single/operations_guide/index#automatically-tuning-osd-memory_ops


Collect Information about the Ceph Cluster
https://support.hpe.com/hpesc/public/docDisplay?docId=a00117940en_us&page=Collect_Information_About_the_Ceph_Cluster.html

Troubleshoot Ceph OSDs Reporting Full
https://support.hpe.com/hpesc/public/docDisplay?docId=a00117940en_us&docLocale=en_US&page=Troubleshoot_Ceph_OSDs_Reporting_Full.html

Troubleshooting Placement Groups
https://access.redhat.com/documentation/en-us/red_hat_ceph_storage/3/html/troubleshooting_guide/troubleshooting-placement-groups


AUTOSCALING PLACEMENT GROUPS
https://docs.ceph.com/en/latest/rados/operations/placement-groups/