rgw源码走读-桶创建流程

Last updated on a year ago

rgw创建桶的流程 也是和其他请求一样，都是三部曲，主要流程在 RGWCreateBucket::execute() 里面，本次也是直接从这里出发（前面处理请求的大差不差，会有专门用一篇文章来介绍），这里组要介绍下bucket的概念，s3对桶的约束，以及解答以下几个问题？

  • 创建桶的时候，桶的实例时持久化哪里？
  • 那个池？是那个对象？有什么内容？桶id 的命名规则是？

创建桶后生成那些文件 ？

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
[root@node86 build]# s3cmd mb s3://hrp
[root@node86 build]# rados  -p  expondata.rgw.meta ls -N root
hrp
.bucket.meta.hrp:7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.1

[root@node86 build]# s3cmd mb s3://hrp1
[root@node86 build]# rados  -p  expondata.rgw.meta ls -N root|sort 
.bucket.meta.hrp1:7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.2
.bucket.meta.hrp:7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.1
hrp
hrp1
# index 池
[root@node86 build]# rados  -p  expondata.rgw.buckets.index  ls 
ceph_context.cc:385:handle_conf_change:WARNING: all dangerous and experimental features are enabled.
.dir.7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.2
.dir.7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.1
[root@node86 build]#


分别创建两个桶可以观察到，每次创建桶都会新增 meta 池中都会新增两个obj，在index新增一个obj，总结下 obj 命名规则

  • .bucket.meta.hrp:7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.1
    • .bucket.meta.[bucket_name]:[桶id]
    • 桶id 构成：[全局唯一的实例id].[num]
      其中num在当前RGW实例中单调递增
  • hrp
    • 桶名字
  • .dir.7fdb0063-3957-4eed-8d4e-c02b208c0d68.4192.1
    • .dir.[bucket_name]:[桶id]
    • 桶id 构成：[全局唯一的实例id].[num]
      其中num在当前RGW实例中单调递增

这些obj作用是？

进入这函数分析下 RGWCreateBucket::execute

对于dir开头的obj，是对应每个bucket shard，在桶初始化时会在index吃 创建shard 数量的obj， bucket shard 的概念会在别的篇幅中介绍RGW Bucket ， 

  RGWCreateBucket::execute()
->RGWRados::create_bucket
->RGWRados::init_bucket_index

对应.bucket.meta 开头的obj ，存储了bucket instance， 看代码流程是 一些桶的关键信息 RGWBucketEntryPoint 结构体中， 然后在 meta池中写入这个oid中
RGWCreateBucket::execute()
->RGWRados::create_bucket
->put_linked_bucket_info
->RGWRados::put_bucket_instance_info
->rgw_bucket_instance_store_info // RGWBucketInfo 写入 obj
->store->meta_mgr->put_entry
->handler->get_pool_and_oid //构造 oid 名字 oid = RGW_BUCKET_INSTANCE_MD_PREFIX + key;

对应桶名字的obj
RGWCreateBucket::execute()
->RGWRados::create_bucket
->put_linked_bucket_info
->RGWRados::put_bucket_instance_info
->put_bucket_entrypoint_info // RGWBucketEntryPoint 写入 obj
->store->meta_mgr->put_entry
->handler->get_pool_and_oid //构造 oid 名字 oid = bukcet那么

这个两个 oid 都在同一个函数实现

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
int RGWRados::put_linked_bucket_info(RGWBucketInfo& info, bool exclusive, real_time mtime, obj_version *pep_objv,
                                     map<string, bufferlist> *pattrs, bool create_entry_point)
{
  bool create_head = !info.has_instance_obj || create_entry_point;
// bucket_instance_info
  int ret = put_bucket_instance_info(info, exclusive, mtime, pattrs);
  if (ret < 0) {
    return ret;
  }

  if (!create_head)
    return 0; /* done! */

  RGWBucketEntryPoint entry_point;
  entry_point.bucket = info.bucket;
  entry_point.owner = info.owner;
  entry_point.creation_time = info.creation_time;
  entry_point.linked = true;
  RGWObjVersionTracker ot;
  if (pep_objv && !pep_objv->tag.empty()) {
    ot.write_version = *pep_objv;
  } else {
    ot.generate_new_write_ver(cct);
    if (pep_objv) {
      *pep_objv = ot.write_version;
    }
  }
//obj: bucekt id
  ret = put_bucket_entrypoint_info(info.bucket.tenant, info.bucket.name, entry_point, exclusive, ot, mtime, NULL); 
  if (ret < 0)
    return ret;

  return 0;
}
  • 这两个obj 有何不同？
    .bucket.meta 开头对象记录很多 bucket 信息（写入了 结构体 RGWBucketInfo），而 bucket name 的对象只存放了简单bucket信息，保存桶名和桶实例id之间的映射， 两者是映射关系

可以用 radosgw-admin metadata 看到这obj的内容

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
[root@node86 build]# radosgw-admin metadata get  bucket.instance:hrp901113311:ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3

{
    "key": "bucket.instance:hrp901113311:ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3",
    "ver": {
        "tag": "_3cPYVcIE4NnQ5KNue_smzoy",
        "ver": 1
    },
    "mtime": "2024-02-06 04:06:18.101843Z",
    "data": {
        "bucket_info": {
            "bucket": {
                "name": "hrp901113311",
                "marker": "ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3",
                "bucket_id": "ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3",
                "tenant": "",
                "explicit_placement": {
                    "data_pool": "",
                    "data_extra_pool": "",
                    "index_pool": ""
                }
            },
            "creation_time": "2024-02-06 04:06:17.695464Z",
            "owner": "testid",
            "flags": 0,
            "zonegroup": "49fb5987-dccb-4aaf-ad9e-3c3437b1f87f",
            "placement_rule": "default-placement/STANDARD",
            "has_instance_obj": "true",
            "quota": {
                "enabled": false,
                "check_on_raw": false,
                "max_size": -1,
                "max_size_kb": 0,
                "max_objects": -1
            },
            "redirection_rules": [],
            "num_shards": 0,
            "bi_shard_hash_type": 0,
            "requester_pays": "false",
            "has_website": "false",
            "swift_versioning": "false",
            "swift_ver_location": "",
            "index_type": 0,
            "mdsearch_config": [],
            "reshard_status": 0,
            "new_bucket_instance_id": "",
            "merge_flag": 1,
            "policy_rule": {
                "policy_id": "",
                "policy_info": [],
                "ctime": "0.000000"
            },
            "obj_lock": {
                "enabled": "true",
                "rule_exist": "false",
                "rule": {
                    "defaultRetention": {
                        "mode": "",
                        "days": 0,
                        "years": 0
                    }
                }
            },
            "version": 7,
            "redirect_mon": "false",
            "tagsearch_config": []
        },
        "attrs": [
            {
                "key": "user.rgw.acl",
                "val": "AgKNAAAAAwIXAAAABgAAAHRlc3RpZAkAAABNLiBUZXN0ZXIEA2oAAAABAQAAAAYAAAB0ZXN0aWQPAAAAAQAAAAYAAAB0ZXN0aWQFAzsAAAACAgQAAAAAAAAABgAAAHRlc3RpZAAAAAAAAAAAAgIEAAAADwAAAAkAAABNLiBUZXN0ZXIAAAAAAAAAAAAAAAAAAAAA"
            }
        ]
    }
}

[root@node86 build]# radosgw-admin metadata get  bucket:hrp901113311

{
    "key": "bucket:hrp901113311",
    "ver": {
        "tag": "_JZbsjMf61290xDFw3HGcofI",
        "ver": 1
    },
    "mtime": "2024-02-06 04:06:18.115677Z",
    "data": {
        "bucket": {
            "name": "hrp901113311",
            "marker": "ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3",
            "bucket_id": "ea296261-93d8-405c-a8a0-eee34d37c6bc.4192.3",
            "tenant": "",
            "explicit_placement": {
                "data_pool": "",
                "data_extra_pool": "",
                "index_pool": ""
            }
        },
        "owner": "testid",
        "creation_time": "2024-02-06 04:06:17.695464Z",
        "linked": "true",
        "has_bucket_info": "false"
    }
}

TODO： 两个obj能否合并在一起？ D版本之后是分开的，分开原因呢？