2 * pNFS Objects layout implementation over open-osd initiator library
4 * Copyright (C) 2009 Panasas Inc. [year of first publication]
7 * Benny Halevy <bhalevy@panasas.com>
8 * Boaz Harrosh <bharrosh@panasas.com>
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of the GNU General Public License version 2
12 * See the file COPYING included with this distribution for more details.
14 * Redistribution and use in source and binary forms, with or without
15 * modification, are permitted provided that the following conditions
18 * 1. Redistributions of source code must retain the above copyright
19 * notice, this list of conditions and the following disclaimer.
20 * 2. Redistributions in binary form must reproduce the above copyright
21 * notice, this list of conditions and the following disclaimer in the
22 * documentation and/or other materials provided with the distribution.
23 * 3. Neither the name of the Panasas company nor the names of its
24 * contributors may be used to endorse or promote products derived
25 * from this software without specific prior written permission.
27 * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED
28 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
29 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
30 * DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
31 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
34 * BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
35 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
36 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
37 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
40 #include <linux/module.h>
41 #include <scsi/osd_initiator.h>
43 #include "objlayout.h"
45 #define NFSDBG_FACILITY NFSDBG_PNFS_LD
47 #define _LLU(x) ((unsigned long long)x)
49 struct objio_dev_ent {
50 struct nfs4_deviceid_node id_node;
55 objio_free_deviceid_node(struct nfs4_deviceid_node *d)
57 struct objio_dev_ent *de = container_of(d, struct objio_dev_ent, id_node);
59 dprintk("%s: free od=%p\n", __func__, de->od);
60 osduld_put_device(de->od);
64 static struct objio_dev_ent *_dev_list_find(const struct nfs_server *nfss,
65 const struct nfs4_deviceid *d_id)
67 struct nfs4_deviceid_node *d;
68 struct objio_dev_ent *de;
70 d = nfs4_find_get_deviceid(nfss->pnfs_curr_ld, nfss->nfs_client, d_id);
74 de = container_of(d, struct objio_dev_ent, id_node);
78 static struct objio_dev_ent *
79 _dev_list_add(const struct nfs_server *nfss,
80 const struct nfs4_deviceid *d_id, struct osd_dev *od,
83 struct nfs4_deviceid_node *d;
84 struct objio_dev_ent *de = kzalloc(sizeof(*de), gfp_flags);
85 struct objio_dev_ent *n;
88 dprintk("%s: -ENOMEM od=%p\n", __func__, od);
92 dprintk("%s: Adding od=%p\n", __func__, od);
93 nfs4_init_deviceid_node(&de->id_node,
99 d = nfs4_insert_deviceid_node(&de->id_node);
100 n = container_of(d, struct objio_dev_ent, id_node);
102 dprintk("%s: Race with other n->od=%p\n", __func__, n->od);
103 objio_free_deviceid_node(&de->id_node);
107 atomic_inc(&de->id_node.ref);
111 struct caps_buffers {
112 u8 caps_key[OSD_CRYPTO_KEYID_SIZE];
113 u8 creds[OSD_CAP_LEN];
116 struct objio_segment {
117 struct pnfs_layout_segment lseg;
119 struct pnfs_osd_object_cred *comps;
122 unsigned stripe_unit;
123 unsigned group_width; /* Data stripe_units without integrity comps */
125 unsigned group_count;
127 unsigned comps_index;
129 /* variable length */
130 struct objio_dev_ent *ods[];
133 static inline struct objio_segment *
134 OBJIO_LSEG(struct pnfs_layout_segment *lseg)
136 return container_of(lseg, struct objio_segment, lseg);
139 /* Send and wait for a get_device_info of devices in the layout,
140 then look them up with the osd_initiator library */
141 static struct objio_dev_ent *_device_lookup(struct pnfs_layout_hdr *pnfslay,
142 struct objio_segment *objio_seg, unsigned comp,
145 struct pnfs_osd_deviceaddr *deviceaddr;
146 struct nfs4_deviceid *d_id;
147 struct objio_dev_ent *ode;
149 struct osd_dev_info odi;
152 d_id = &objio_seg->comps[comp].oc_object_id.oid_device_id;
154 ode = _dev_list_find(NFS_SERVER(pnfslay->plh_inode), d_id);
158 err = objlayout_get_deviceinfo(pnfslay, d_id, &deviceaddr, gfp_flags);
160 dprintk("%s: objlayout_get_deviceinfo dev(%llx:%llx) =>%d\n",
161 __func__, _DEVID_LO(d_id), _DEVID_HI(d_id), err);
165 odi.systemid_len = deviceaddr->oda_systemid.len;
166 if (odi.systemid_len > sizeof(odi.systemid)) {
169 } else if (odi.systemid_len)
170 memcpy(odi.systemid, deviceaddr->oda_systemid.data,
172 odi.osdname_len = deviceaddr->oda_osdname.len;
173 odi.osdname = (u8 *)deviceaddr->oda_osdname.data;
175 if (!odi.osdname_len && !odi.systemid_len) {
176 dprintk("%s: !odi.osdname_len && !odi.systemid_len\n",
182 od = osduld_info_lookup(&odi);
183 if (unlikely(IS_ERR(od))) {
185 dprintk("%s: osduld_info_lookup => %d\n", __func__, err);
189 ode = _dev_list_add(NFS_SERVER(pnfslay->plh_inode), d_id, od,
193 dprintk("%s: return=%d\n", __func__, err);
194 objlayout_put_deviceinfo(deviceaddr);
195 return err ? ERR_PTR(err) : ode;
198 static int objio_devices_lookup(struct pnfs_layout_hdr *pnfslay,
199 struct objio_segment *objio_seg,
205 /* lookup all devices */
206 for (i = 0; i < objio_seg->num_comps; i++) {
207 struct objio_dev_ent *ode;
209 ode = _device_lookup(pnfslay, objio_seg, i, gfp_flags);
210 if (unlikely(IS_ERR(ode))) {
214 objio_seg->ods[i] = ode;
219 dprintk("%s: return=%d\n", __func__, err);
223 static int _verify_data_map(struct pnfs_osd_layout *layout)
225 struct pnfs_osd_data_map *data_map = &layout->olo_map;
229 /* FIXME: Only raid0 for now. if not go through MDS */
230 if (data_map->odm_raid_algorithm != PNFS_OSD_RAID_0) {
231 printk(KERN_ERR "Only RAID_0 for now\n");
234 if (0 != (data_map->odm_num_comps % (data_map->odm_mirror_cnt + 1))) {
235 printk(KERN_ERR "Data Map wrong, num_comps=%u mirrors=%u\n",
236 data_map->odm_num_comps, data_map->odm_mirror_cnt);
240 if (data_map->odm_group_width)
241 group_width = data_map->odm_group_width;
243 group_width = data_map->odm_num_comps /
244 (data_map->odm_mirror_cnt + 1);
246 stripe_length = (u64)data_map->odm_stripe_unit * group_width;
247 if (stripe_length >= (1ULL << 32)) {
248 printk(KERN_ERR "Total Stripe length(0x%llx)"
249 " >= 32bit is not supported\n", _LLU(stripe_length));
253 if (0 != (data_map->odm_stripe_unit & ~PAGE_MASK)) {
254 printk(KERN_ERR "Stripe Unit(0x%llx)"
255 " must be Multples of PAGE_SIZE(0x%lx)\n",
256 _LLU(data_map->odm_stripe_unit), PAGE_SIZE);
263 static void copy_single_comp(struct pnfs_osd_object_cred *cur_comp,
264 struct pnfs_osd_object_cred *src_comp,
265 struct caps_buffers *caps_p)
267 WARN_ON(src_comp->oc_cap_key.cred_len > sizeof(caps_p->caps_key));
268 WARN_ON(src_comp->oc_cap.cred_len > sizeof(caps_p->creds));
270 *cur_comp = *src_comp;
272 memcpy(caps_p->caps_key, src_comp->oc_cap_key.cred,
273 sizeof(caps_p->caps_key));
274 cur_comp->oc_cap_key.cred = caps_p->caps_key;
276 memcpy(caps_p->creds, src_comp->oc_cap.cred,
277 sizeof(caps_p->creds));
278 cur_comp->oc_cap.cred = caps_p->creds;
281 int objio_alloc_lseg(struct pnfs_layout_segment **outp,
282 struct pnfs_layout_hdr *pnfslay,
283 struct pnfs_layout_range *range,
284 struct xdr_stream *xdr,
287 struct objio_segment *objio_seg;
288 struct pnfs_osd_xdr_decode_layout_iter iter;
289 struct pnfs_osd_layout layout;
290 struct pnfs_osd_object_cred *cur_comp, src_comp;
291 struct caps_buffers *caps_p;
294 err = pnfs_osd_xdr_decode_layout_map(&layout, &iter, xdr);
298 err = _verify_data_map(&layout);
302 objio_seg = kzalloc(sizeof(*objio_seg) +
303 sizeof(objio_seg->ods[0]) * layout.olo_num_comps +
304 sizeof(*objio_seg->comps) * layout.olo_num_comps +
305 sizeof(struct caps_buffers) * layout.olo_num_comps,
310 objio_seg->comps = (void *)(objio_seg->ods + layout.olo_num_comps);
311 cur_comp = objio_seg->comps;
312 caps_p = (void *)(cur_comp + layout.olo_num_comps);
313 while (pnfs_osd_xdr_decode_layout_comp(&src_comp, &iter, xdr, &err))
314 copy_single_comp(cur_comp++, &src_comp, caps_p++);
318 objio_seg->num_comps = layout.olo_num_comps;
319 objio_seg->comps_index = layout.olo_comps_index;
320 err = objio_devices_lookup(pnfslay, objio_seg, gfp_flags);
324 objio_seg->mirrors_p1 = layout.olo_map.odm_mirror_cnt + 1;
325 objio_seg->stripe_unit = layout.olo_map.odm_stripe_unit;
326 if (layout.olo_map.odm_group_width) {
327 objio_seg->group_width = layout.olo_map.odm_group_width;
328 objio_seg->group_depth = layout.olo_map.odm_group_depth;
329 objio_seg->group_count = layout.olo_map.odm_num_comps /
330 objio_seg->mirrors_p1 /
331 objio_seg->group_width;
333 objio_seg->group_width = layout.olo_map.odm_num_comps /
334 objio_seg->mirrors_p1;
335 objio_seg->group_depth = -1;
336 objio_seg->group_count = 1;
339 *outp = &objio_seg->lseg;
344 dprintk("%s: Error: return %d\n", __func__, err);
349 void objio_free_lseg(struct pnfs_layout_segment *lseg)
352 struct objio_segment *objio_seg = OBJIO_LSEG(lseg);
354 for (i = 0; i < objio_seg->num_comps; i++) {
355 if (!objio_seg->ods[i])
357 nfs4_put_deviceid_node(&objio_seg->ods[i]->id_node);
363 static struct pnfs_layoutdriver_type objlayout_type = {
364 .id = LAYOUT_OSD2_OBJECTS,
365 .name = "LAYOUT_OSD2_OBJECTS",
367 .alloc_layout_hdr = objlayout_alloc_layout_hdr,
368 .free_layout_hdr = objlayout_free_layout_hdr,
370 .alloc_lseg = objlayout_alloc_lseg,
371 .free_lseg = objlayout_free_lseg,
373 .free_deviceid_node = objio_free_deviceid_node,
376 MODULE_DESCRIPTION("pNFS Layout Driver for OSD2 objects");
377 MODULE_AUTHOR("Benny Halevy <bhalevy@panasas.com>");
378 MODULE_LICENSE("GPL");
383 int ret = pnfs_register_layoutdriver(&objlayout_type);
387 "%s: Registering OSD pNFS Layout Driver failed: error=%d\n",
390 printk(KERN_INFO "%s: Registered OSD pNFS Layout Driver\n",
398 pnfs_unregister_layoutdriver(&objlayout_type);
399 printk(KERN_INFO "%s: Unregistered OSD pNFS Layout Driver\n",
403 module_init(objlayout_init);
404 module_exit(objlayout_exit);