Ask Your Question

Revision history [back]

click to hide/show revision 1
initial version

[ceph] Does replication in osds and pools add up?

Hi,

I am currently investigating the usage of our ceph storage and I think it might be that I made a mistake withe the replication. I have three physical nodes with six disks each. Is it possible that i have an absolute replication count of nine over the whole cluster? Each object three times in the pool and the pool three times over the osds? And if that is the case. What would be the best way to fix this.

Thanks for your help.

root@DIR2:/home/ubuntu# docker exec -ti ceph_mon rados df
POOL_NAME                 USED    OBJECTS CLONES COPIES  MISSING_ON_PRIMARY UNFOUND DEGRADED RD_OPS     RD      WR_OPS     WR
.rgw.root                 3.08KiB       8      0      24                  0       0        0        510  340KiB          8    8KiB
backups                       19B       2      0       6                  0       0        0        314  241KiB      84514  110GiB
default.rgw.buckets.data  72.3MiB     523      0    1569                  0       0        0       1350 57.4MiB       4946 72.5MiB
default.rgw.buckets.index      0B       1      0       3                  0       0        0       3229 4.60MiB       2059      0B
default.rgw.control            0B       8      0      24                  0       0        0          0      0B          0      0B
default.rgw.log                0B     207      0     621                  0       0        0   79766049 76.1GiB   53150230      0B
default.rgw.meta             780B       4      0      12                  0       0        0       1107 1018KiB         84   13KiB
gnocchi                   1.56GiB  128638      0  385914                  0       0        0  506847789  871GiB  546727389  242GiB
images                    35.1GiB    4536      0   13608                  0       0        0     292167  114GiB     105460 98.8GiB
vms                       70.6GiB   18328      0   54984                  0       0        0 1418007932 75.7TiB 1283170530 11.8TiB
volumes                   3.05TiB  802897   2489 2408691                  0       0        0 1259978578 20.3TiB 1200805704 45.0TiB

total_objects    955152
total_used       9.44TiB
total_avail      5.20TiB
total_space      14.6TiB




root@DIR2:/home/ubuntu# docker exec -ti ceph_mon rbd -p volumes du
NAME                                                                                      PROVISIONED    USED
volume-083f6e89-8ae5-4fde-908d-e52241cd0537@snapshot-5a98b309-313e-4ccf-8113-e0353c3ecf74       40GiB 19.4GiB
volume-083f6e89-8ae5-4fde-908d-e52241cd0537                                                     40GiB 19.6GiB
volume-12d0cba0-e2ec-4e5b-9dd2-fd1e36e03663@snapshot-79ad1e0f-1c91-4104-8490-eadeee28c572       40GiB 37.5GiB
volume-12d0cba0-e2ec-4e5b-9dd2-fd1e36e03663                                                     40GiB 37.1GiB
volume-24a65e11-1432-4f87-9917-ebbb6d96574d                                                     20GiB 9.82GiB
volume-3f76f808-7870-4e4e-9b74-e2225f1f6594                                                     61GiB 53.3GiB
volume-46a03999-c012-4565-8522-948a86f4175f                                                     35GiB 5.17GiB
volume-4cc292e1-42f1-4060-a520-b46fee51640e                                                    300GiB 76.1GiB
volume-4d44c495-213f-4265-ac3c-a3fe4b43f642                                                     40GiB 36.7GiB
volume-582da869-6e89-49f5-b493-467646eb3d46                                                     30GiB 16.7GiB
volume-6f22fdb0-e3c9-4f7d-9e5c-bbf2d53cd2cf                                                     30GiB 5.38GiB
volume-6f3440bc-6360-4cd6-97f7-1d60ef350280                                                     40GiB 23.8GiB
volume-75c84d73-86cc-44d7-8037-9db46b28444f@snapshot-cc1e98c0-e6a5-4694-805f-7612be34251a       35GiB 32.3GiB
volume-75c84d73-86cc-44d7-8037-9db46b28444f                                                     35GiB      0B
volume-7d2a4d7c-7634-47ad-92f3-e776400f2141                                                     10GiB 9.97GiB
volume-83fadc2f-7bf2-447a-838a-2fcd07622255                                                     80GiB 59.0GiB
volume-84bf07fb-2d90-4cd6-bce1-7d348c43677a                                                     40GiB 13.8GiB
volume-987c4a60-3a4f-4842-b097-4888e48b6158                                                     50GiB 37.6GiB
volume-a08780fd-169c-4382-9704-05f75877eac0                                                     30GiB 23.5GiB
volume-a2a60be3-71ff-46af-a277-3e32169a138a                                                    300GiB  294GiB
volume-a3db896f-3655-41a0-b33f-839f53fab082                                                     30GiB 21.3GiB
volume-ac44bea6-917d-4840-b64c-ebc10bb0f587                                                     40GiB 19.0GiB
volume-aeed382c-0de7-4912-9f14-7511867a1b92                                                     50GiB 30.0GiB
volume-b07efa2f-4955-4488-8bee-91574bd0b699                                                    150GiB  142GiB
volume-bb5049fc-edee-4c85-9d3b-21dd832600da                                                      5GiB 4.86GiB
volume-bf45d75e-38c5-48b5-966e-e81436094923                                                    300GiB 12.4GiB
volume-c9667c0b-50a1-49f0-9c8b-3abbe0d52478                                                     50GiB 39.5GiB
volume-e9d81955-4c00-4b9f-ab5f-23d658daf055                                                     40GiB 37.7GiB
<TOTAL>                                                                                       1.80TiB 1.09TiB











root@DIR2:/home/ubuntu# docker exec -ti ceph_mon ceph osd dump
epoch 8968
fsid 51276edb-2b73-4610-8a62-1d028a9ba8d6
created 2018-10-05 19:28:29.139438
modified 2019-12-03 08:19:45.327152
flags sortbitwise,recovery_deletes,purged_snapdirs
crush_version 131
full_ratio 0.95
backfillfull_ratio 0.9
nearfull_ratio 0.67
require_min_compat_client jewel
min_compat_client jewel
require_osd_release luminous
pool 1 '.rgw.root' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 2 'images' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 128 pgp_num 128 last_change 8516 lfor 0/4732 flags hashpspool,nearfull stripe_width 0 application rbd
    removed_snaps [1~17,19~4,1e~f,2e~2,31~14,46~30,77~22,9a~1]
pool 3 'volumes' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 512 pgp_num 512 last_change 8966 lfor 0/3792 flags hashpspool,nearfull stripe_width 0 application rbd
    removed_snaps [1~10,12~1,14~7,1c~23]
pool 4 'backups' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 50 pgp_num 50 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rbd
    removed_snaps [1~3]
pool 5 'default.rgw.control' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 6 'default.rgw.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 7 'default.rgw.log' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 8 'vms' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 50 pgp_num 50 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rbd
    removed_snaps [1~3]
pool 9 'gnocchi' replicated size 3 min_size 2 crush_rule 1 object_hash rjenkins pg_num 50 pgp_num 50 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 10 'default.rgw.buckets.index' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
pool 11 'default.rgw.buckets.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 last_change 8513 flags hashpspool,nearfull stripe_width 0 application rgw
max_osd 24
osd.0 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3105,8440) 172.31.0.6:6800/63732 172.31.0.6:6802/1063732 172.31.0.6:6803/1063732 172.31.0.6:6812/1063732 exists,nearfull,up 8fcc489f-ebc5-4402-8c07-06a140a8947b
osd.1 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3106,8438) 172.31.0.5:6800/13010 172.31.0.5:6824/4013010 172.31.0.5:6825/4013010 172.31.0.5:6826/4013010 exists,up 7abfd154-befb-4f9a-9dd4-3cf153d3c660
osd.2 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [6442,8460) 172.31.0.7:6820/27654 172.31.0.7:6801/1027654 172.31.0.7:6802/1027654 172.31.0.7:6803/1027654 exists,nearfull,up 2737aa42-2b42-42ad-b43a-f745a574b187
osd.3 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3110,8440) 172.31.0.5:6804/13151 172.31.0.5:6817/4013151 172.31.0.5:6818/4013151 172.31.0.5:6819/4013151 exists,up 7650efaa-e376-4154-b90b-83be7e9e8da8
osd.4 up   in  weight 1 up_from 3408 up_thru 8463 down_at 3404 last_clean_interval [3114,3407) 172.31.0.5:6808/13293 172.31.0.5:6820/3013293 172.31.0.5:6821/3013293 172.31.0.5:6822/3013293 exists,up 0c3e0964-6063-4a29-8faa-776d89ca2ebc
osd.6 up   in  weight 1 up_from 3107 up_thru 8463 down_at 3102 last_clean_interval [878,3101) 172.31.0.6:6804/63913 172.31.0.6:6813/63913 172.31.0.6:6814/63913 172.31.0.6:6815/63913 exists,up 4fcee0a2-6527-416b-a009-a82234c9a701
osd.8 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3114,8460) 172.31.0.5:6801/13473 172.31.0.5:6813/4013473 172.31.0.5:6814/4013473 172.31.0.5:6815/4013473 exists,nearfull,up 861da0f9-d8e3-46a7-aaaf-27ddb47ac952
osd.10 up   in  weight 1 up_from 3408 up_thru 8463 down_at 3404 last_clean_interval [3175,3407) 172.31.0.5:6816/8281 172.31.0.5:6809/3008281 172.31.0.5:6810/3008281 172.31.0.5:6811/3008281 exists,up 66f175e5-97ce-4dcc-9d99-aa421fc86273
osd.11 up   in  weight 1 up_from 3113 up_thru 8463 down_at 3107 last_clean_interval [879,3106) 172.31.0.6:6805/64053 172.31.0.6:6806/64053 172.31.0.6:6807/64053 172.31.0.6:6808/64053 exists,up 58a528e9-88ee-432a-b4b8-96ee36d82226
osd.12 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3114,8438) 172.31.0.6:6801/64196 172.31.0.6:6821/1064196 172.31.0.6:6822/1064196 172.31.0.6:6823/1064196 exists,up f892d647-6b8d-46f0-8022-55a414971b84
osd.13 up   in  weight 1 up_from 8462 up_thru 8463 down_at 8437 last_clean_interval [3176,8440) 172.31.0.6:6816/56333 172.31.0.6:6809/1056333 172.31.0.6:6810/1056333 172.31.0.6:6811/1056333 exists,up da582b39-33c1-4e18-9f55-fda25ebb3a1a
osd.15 up   in  weight 1 up_from 3577 up_thru 8463 down_at 0 last_clean_interval [0,0) 172.31.0.5:6805/55263 172.31.0.5:6806/55263 172.31.0.5:6807/55263 172.31.0.5:6823/55263 exists,up faa818cc-225b-4567-896a-6af7453b0078
osd.16 up   in  weight 1 up_from 3836 up_thru 8463 down_at 3829 last_clean_interval [3577,3835) 172.31.0.6:6820/31177 172.31.0.6:6824/1031177 172.31.0.6:6825/1031177 172.31.0.6:6826/1031177 exists,up e2826db4-7d6d-4a8a-a628-890848b6a048
osd.18 up   in  weight 1 up_from 7058 up_thru 8463 down_at 6934 last_clean_interval [4912,6936) 172.31.0.7:6800/35953 172.31.0.7:6809/1035953 172.31.0.7:6810/1035953 172.31.0.7:6811/1035953 exists,up 566a6caf-6265-48d6-9165-e0d99625c4bb
osd.19 up   in  weight 1 up_from 4914 up_thru 8463 down_at 0 last_clean_interval [0,0) 172.31.0.7:6804/36088 172.31.0.7:6805/36088 172.31.0.7:6806/36088 172.31.0.7:6807/36088 exists,up 15539095-9ee6-43ff-88d3-8e1e044a388f
osd.20 up   in  weight 1 up_from 7058 up_thru 8463 down_at 6934 last_clean_interval [4917,6936) 172.31.0.7:6808/36261 172.31.0.7:6824/1036261 172.31.0.7:6825/1036261 172.31.0.7:6826/1036261 exists,up 9cdde56b-356d-44dc-bfc5-cd03db2d357c
osd.21 up   in  weight 1 up_from 4917 up_thru 8463 down_at 0 last_clean_interval [0,0) 172.31.0.7:6812/36396 172.31.0.7:6813/36396 172.31.0.7:6814/36396 172.31.0.7:6815/36396 exists,up 74e0c5ea-51f7-4588-ab21-580da729bb42
osd.22 up   in  weight 1 up_from 8440 up_thru 8463 down_at 8438 last_clean_interval [4919,8437) 172.31.0.7:6816/23943 172.31.0.7:6817/23943 172.31.0.7:6818/23943 172.31.0.7:6819/23943 exists,up b2d171af-5d83-4e14-9090-b6b66624e4c5