Robbat2 (robbat2) wrote,
Robbat2
robbat2

Ceph RGW Hammer->Jewel upgrade: adding realms, periods etc

Some quick notes on upgrading a Hammer-era Ceph RGW setup to Jewel, because the upstream notes don't cover it well. The multisite docs are the closest there is, but here's what I put together instead.

  • The Zone concept has remained the same.
  • A Region is now a Zonegroup.
  • The top-level RegionMap is moved inside the content of a Period
  • Only one Period can be live at a time, and changes are made to a non-live Period
  • The Realm describes which Period is live.
  • Additionally, there can be a default Zonegroup and Zone inside the period, as well as a default Zone inside a Zonegroup.

Initial state, if you were to look on Hammer:
# radosgw-admin region list
{
  "default_info": {
    "default_region": "default"
  },
  "regions": [
    "default"
  ]
}
# radosgw-admin region-map get
{
  "master_region": "default",
  "bucket_quota": {
    "max_objects": -1,
    "enabled": false,
    "max_size_kb": -1
  },
  "user_quota": {
    "max_objects": -1,
    "enabled": false,
    "max_size_kb": -1
  },
  "regions": [
    {
      "val": {
        "zones": [
          {
            "name": "default",
            "log_meta": "false",
            "endpoints": [

            ],
            "bucket_index_max_shards": 31,
            "log_data": "false"
          }
        ],
        "name": "default",
        "endpoints": [
	      "https://CENSORED-1.EXAMPLE.COM",
	      "https://CENSORED-2.EXAMPLE.COM"
        ],
        "api_name": "CENSORED",
        "default_placement": "default-placement",
        "is_master": "true",
        "hostnames": [
	      "CENSORED-1.EXAMPLE.COM",
	      "CENSORED-2.EXAMPLE.COM"
        ],
        "placement_targets": [
          {
            "name": "default-placement",
            "tags": [

            ]
          }
        ],
        "master_zone": ""
      },
      "key": "default"
    }
  ]
}
# radosgw-admin region get --rgw-region=default
{
  "zones": [
    {
      "log_meta": "false",
      "name": "default",
      "bucket_index_max_shards": 31,
      "endpoints": [

      ],
      "log_data": "false"
    }
  ],
  "master_zone": "",
  "is_master": "true",
  "placement_targets": [
    {
      "name": "default-placement",
      "tags": [

      ]
    }
  ],
  "default_placement": "default-placement",
  "name": "default",
  "hostnames": [
	"CENSORED-1.EXAMPLE.COM",
	"CENSORED-2.EXAMPLE.COM"
  ],
  "endpoints": [
    "https://CENSORED-1.EXAMPLE.COM",
    "https://CENSORED-2.EXAMPLE.COM"
  ],
  "api_name": "CENSORED"
}
# radosgw-admin zone get --rgw-region=default --rgw-zone=default
{
  "log_pool": ".log",
  "user_swift_pool": ".users.swift",
  "placement_pools": [
    {
      "val": {
        "data_pool": ".rgw.buckets",
        "data_extra_pool": ".rgw.buckets.extra",
        "index_pool": ".rgw.buckets.index"
      },
      "key": "default-placement"
    }
  ],
  "user_keys_pool": ".users",
  "control_pool": ".rgw.control",
  "domain_root": ".rgw",
  "usage_log_pool": ".usage",
  "gc_pool": ".rgw.gc",
  "system_key": {
    "access_key": "",
    "secret_key": ""
  },
  "intent_log_pool": ".intent-log",
  "user_uid_pool": ".users.uid",
  "user_email_pool": ".users.email"
}


Initial state, if you were to look on Jewel:
# radosgw-admin zone list
{
    "default_info": "",
    "zones": [
        "default"
    ]
}
# radosgw-admin zonegroup list
{
    "default_info": "",
    "zonegroups": [
        "default"
    ]
}
# TODO: fill the rest of this up.

# Now changing stuff up:
# export SYSTEM_ACCESS_KEY=... SYSTEM_SECRET_KEY=...
# radosgw-admin user create \
  --system  
  --uid=zone.user \
  --display-name="Zone User" \
  --access-key=$SYSTEM_ACCESS_KEY \
  --secret=$SYSTEM_SECRET_KEY
{
  "user_id": "zone.user",
  "display_name": "Zone User",
  "email": "",
  "suspended": 0,
  "max_buckets": 1000,
  "auid": 0,
  "subusers": [],
  "keys": [
      {
          "user": "zone.user",
          "access_key": "...",
          "secret_key": "..."
      }
  ],
  "swift_keys": [],
  "caps": [],
  "op_mask": "read, write, delete",
  "system": "true",
  "default_placement": "",
  "placement_tags": [],
  "bucket_quota": {
      "enabled": false,
      "max_size_kb": -1,
      "max_objects": -1
  },
  "user_quota": {
      "enabled": false,
      "max_size_kb": -1,
      "max_objects": -1
  },
  "temp_url_keys": []
}


# radosgw-admin realm create --rgw-realm gold
{
    "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "name": "gold",
    "current_period": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "epoch": 1
}


# radosgw-admin realm list
{
    "default_info": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realms": [
        "gold"
    ]
}


# radosgw-admin realm get
{
    "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "name": "gold",
    "current_period": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "epoch": 1
}


# radosgw-admin period list
{
    "periods": [
        "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb"
    ]
}


# radosgw-admin period get
{
    "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "epoch": 1,
    "predecessor_uuid": "",
    "sync_status": [],
    "period_map": {
        "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
        "zonegroups": [],
        "short_zone_ids": []
    },
    "master_zonegroup": "",
    "master_zone": "",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realm_name": "gold",
    "realm_epoch": 1
}


# radosgw-admin period update --master-zone=default --master-zonegroup=default
{
    "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103:staging",
    "epoch": 1,
    "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "sync_status": [],
    "period_map": {
        "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
        "zonegroups": [],
        "short_zone_ids": []
    },
    "master_zonegroup": "",
    "master_zone": "",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realm_name": "gold",
    "realm_epoch": 2
}


# radosgw-admin period prepare
{
    "id": "8fb1cfbc-ad63-4d92-886a-d939cc52862b",
    "epoch": 1,
    "predecessor_uuid": "",
    "sync_status": [],
    "period_map": {
        "id": "8fb1cfbc-ad63-4d92-886a-d939cc52862b",
        "zonegroups": [],
        "short_zone_ids": []
    },
    "master_zonegroup": "",
    "master_zone": "",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realm_name": "gold",
    "realm_epoch": 1
}

# radosgw-admin zone get --rgw-zonegroup=default --rgw-zone=default >zone.json
# radosgw-admin zonegroup get --rgw-zonegroup=default --rgw-zone=default >zonegroup.json
# $EDITOR zonegroup.json zone.json
## Add the following data:
## both files: Set realm_id
## zone.json: Set system_user.access_key, Set system_user.secret_key
## zonegroup.json: Set master_zone to "default", Set is_master to "true".
# radosgw-admin zone set --rgw-zone=default --rgw-zonegroup=default \
  --realm-id=1ac4fd8d-9e77-4fd2-ad54-b591f1734103 \
  --infile zone.json \
  --master --default
# radosgw-admin zonegroup set --rgw-zonegroup=default \
  --realm-id=1ac4fd8d-9e77-4fd2-ad54-b591f1734103 \
  --infile zonegroup.json \
  --master --default


# radosgw-admin period update
{
    "id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103:staging",
    "epoch": 1,
    "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "sync_status": [],
    "period_map": {
        "id": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
        "zonegroups": [
            {
                "id": "default",
                "name": "default",
                "api_name": "CENSORED",
                "is_master": "true",
                "endpoints": [
                    "https:\/\/CENSORED-1.EXAMPLE.COM",
                    "https:\/\/CENSORED-2.EXAMPLE.COM"
                ],
                "hostnames": [
                    "CENSORED-1.EXAMPLE.COM",
                    "CENSORED-2.EXAMPLE.COM"
                ],
                "hostnames_s3website": [],
                "master_zone": "default",
                "zones": [
                    {
                        "id": "default",
                        "name": "default",
                        "endpoints": [],
                        "log_meta": "true",
                        "log_data": "false",
                        "bucket_index_max_shards": 31,
                        "read_only": "false"
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": []
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103"
            }
        ],
        "short_zone_ids": [
            {
                "key": "default",
                "val": 2610307010
            }
        ]
    },
    "master_zonegroup": "default",
    "master_zone": "default",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realm_name": "gold",
    "realm_epoch": 2
}


# radosgw-admin period commit
2016-08-16 17:51:22.324368 7f8562da6900  0 error read_lastest_epoch .rgw.root:periods.8d0d4955-592c-48b5-93d1-3fa1cec17579.latest_epoch
2016-08-16 17:51:22.347375 7f8562da6900  1 Set the period's master zonegroup default as the default
{
    "id": "8d0d4955-592c-48b5-93d1-3fa1cec17579",
    "epoch": 1,
    "predecessor_uuid": "f8fafae9-b6d2-41f6-b7aa-7b03fea57bfb",
    "sync_status": [
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        "",
        ""
    ],
    "period_map": {
        "id": "8d0d4955-592c-48b5-93d1-3fa1cec17579",
        "zonegroups": [
            {
                "id": "default",
                "name": "default",
                "api_name": "CENSORED",
                "is_master": "true",
                "endpoints": [
                    "https:\/\/CENSORED-1.EXAMPLE.COM",
                    "https:\/\/CENSORED-2.EXAMPLE.COM"
                ],
                "hostnames": [
                    "CENSORED-1.EXAMPLE.COM",
                    "CENSORED-2.EXAMPLE.COM"
                ],
                "hostnames_s3website": [],
                "master_zone": "default",
                "zones": [
                    {
                        "id": "default",
                        "name": "default",
                        "endpoints": [],
                        "log_meta": "true",
                        "log_data": "false",
                        "bucket_index_max_shards": 31,
                        "read_only": "false"
                    }
                ],
                "placement_targets": [
                    {
                        "name": "default-placement",
                        "tags": []
                    }
                ],
                "default_placement": "default-placement",
                "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103"
            }
        ],
        "short_zone_ids": [
            {
                "key": "default",
                "val": 2610307010
            }
        ]
    },
    "master_zonegroup": "default",
    "master_zone": "default",
    "period_config": {
        "bucket_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        },
        "user_quota": {
            "enabled": false,
            "max_size_kb": -1,
            "max_objects": -1
        }
    },
    "realm_id": "1ac4fd8d-9e77-4fd2-ad54-b591f1734103",
    "realm_name": "gold",
    "realm_epoch": 2
}


Tags: ceph, hammer, jewel, rgw, upgrade
  • Post a new comment

    Error

    Anonymous comments are disabled in this journal

    default userpic

    Your reply will be screened

    Your IP address will be recorded 

  • 0 comments