xref: /qemu/hw/virtio/vhost-backend.c (revision c471ad0e9bd46ca5f5c9c796e727230e043a091d)
11a1bfac9SNikolay Nikolaev /*
21a1bfac9SNikolay Nikolaev  * vhost-backend
31a1bfac9SNikolay Nikolaev  *
41a1bfac9SNikolay Nikolaev  * Copyright (c) 2013 Virtual Open Systems Sarl.
51a1bfac9SNikolay Nikolaev  *
61a1bfac9SNikolay Nikolaev  * This work is licensed under the terms of the GNU GPL, version 2 or later.
71a1bfac9SNikolay Nikolaev  * See the COPYING file in the top-level directory.
81a1bfac9SNikolay Nikolaev  *
91a1bfac9SNikolay Nikolaev  */
101a1bfac9SNikolay Nikolaev 
119b8bfe21SPeter Maydell #include "qemu/osdep.h"
12a9c94277SMarkus Armbruster #include <linux/vhost.h>
13a9c94277SMarkus Armbruster #include <sys/ioctl.h>
141a1bfac9SNikolay Nikolaev #include "hw/virtio/vhost.h"
151a1bfac9SNikolay Nikolaev #include "hw/virtio/vhost-backend.h"
161a1bfac9SNikolay Nikolaev #include "qemu/error-report.h"
171a1bfac9SNikolay Nikolaev 
181a1bfac9SNikolay Nikolaev static int vhost_kernel_call(struct vhost_dev *dev, unsigned long int request,
191a1bfac9SNikolay Nikolaev                              void *arg)
201a1bfac9SNikolay Nikolaev {
211a1bfac9SNikolay Nikolaev     int fd = (uintptr_t) dev->opaque;
221a1bfac9SNikolay Nikolaev 
231a1bfac9SNikolay Nikolaev     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
241a1bfac9SNikolay Nikolaev 
251a1bfac9SNikolay Nikolaev     return ioctl(fd, request, arg);
261a1bfac9SNikolay Nikolaev }
271a1bfac9SNikolay Nikolaev 
281a1bfac9SNikolay Nikolaev static int vhost_kernel_init(struct vhost_dev *dev, void *opaque)
291a1bfac9SNikolay Nikolaev {
301a1bfac9SNikolay Nikolaev     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
311a1bfac9SNikolay Nikolaev 
321a1bfac9SNikolay Nikolaev     dev->opaque = opaque;
331a1bfac9SNikolay Nikolaev 
341a1bfac9SNikolay Nikolaev     return 0;
351a1bfac9SNikolay Nikolaev }
361a1bfac9SNikolay Nikolaev 
371a1bfac9SNikolay Nikolaev static int vhost_kernel_cleanup(struct vhost_dev *dev)
381a1bfac9SNikolay Nikolaev {
391a1bfac9SNikolay Nikolaev     int fd = (uintptr_t) dev->opaque;
401a1bfac9SNikolay Nikolaev 
411a1bfac9SNikolay Nikolaev     assert(dev->vhost_ops->backend_type == VHOST_BACKEND_TYPE_KERNEL);
421a1bfac9SNikolay Nikolaev 
431a1bfac9SNikolay Nikolaev     return close(fd);
441a1bfac9SNikolay Nikolaev }
451a1bfac9SNikolay Nikolaev 
462ce68e4cSIgor Mammedov static int vhost_kernel_memslots_limit(struct vhost_dev *dev)
472ce68e4cSIgor Mammedov {
482ce68e4cSIgor Mammedov     int limit = 64;
492ce68e4cSIgor Mammedov     char *s;
502ce68e4cSIgor Mammedov 
512ce68e4cSIgor Mammedov     if (g_file_get_contents("/sys/module/vhost/parameters/max_mem_regions",
522ce68e4cSIgor Mammedov                             &s, NULL, NULL)) {
532ce68e4cSIgor Mammedov         uint64_t val = g_ascii_strtoull(s, NULL, 10);
542ce68e4cSIgor Mammedov         if (!((val == G_MAXUINT64 || !val) && errno)) {
552ce68e4cSIgor Mammedov             return val;
562ce68e4cSIgor Mammedov         }
572ce68e4cSIgor Mammedov         error_report("ignoring invalid max_mem_regions value in vhost module:"
582ce68e4cSIgor Mammedov                      " %s", s);
592ce68e4cSIgor Mammedov     }
602ce68e4cSIgor Mammedov     return limit;
612ce68e4cSIgor Mammedov }
622ce68e4cSIgor Mammedov 
6321e70425SMarc-André Lureau static int vhost_kernel_net_set_backend(struct vhost_dev *dev,
6421e70425SMarc-André Lureau                                         struct vhost_vring_file *file)
6521e70425SMarc-André Lureau {
6621e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_NET_SET_BACKEND, file);
6721e70425SMarc-André Lureau }
6821e70425SMarc-André Lureau 
6921e70425SMarc-André Lureau static int vhost_kernel_scsi_set_endpoint(struct vhost_dev *dev,
7021e70425SMarc-André Lureau                                           struct vhost_scsi_target *target)
7121e70425SMarc-André Lureau {
7221e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SCSI_SET_ENDPOINT, target);
7321e70425SMarc-André Lureau }
7421e70425SMarc-André Lureau 
7521e70425SMarc-André Lureau static int vhost_kernel_scsi_clear_endpoint(struct vhost_dev *dev,
7621e70425SMarc-André Lureau                                             struct vhost_scsi_target *target)
7721e70425SMarc-André Lureau {
7821e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SCSI_CLEAR_ENDPOINT, target);
7921e70425SMarc-André Lureau }
8021e70425SMarc-André Lureau 
8121e70425SMarc-André Lureau static int vhost_kernel_scsi_get_abi_version(struct vhost_dev *dev, int *version)
8221e70425SMarc-André Lureau {
8321e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SCSI_GET_ABI_VERSION, version);
8421e70425SMarc-André Lureau }
8521e70425SMarc-André Lureau 
8621e70425SMarc-André Lureau static int vhost_kernel_set_log_base(struct vhost_dev *dev, uint64_t base,
879a78a5ddSMarc-André Lureau                                      struct vhost_log *log)
88c2bea314SMarc-André Lureau {
89c2bea314SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_LOG_BASE, &base);
90c2bea314SMarc-André Lureau }
91c2bea314SMarc-André Lureau 
9221e70425SMarc-André Lureau static int vhost_kernel_set_mem_table(struct vhost_dev *dev,
9321e70425SMarc-André Lureau                                       struct vhost_memory *mem)
9421e70425SMarc-André Lureau {
9521e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_MEM_TABLE, mem);
9621e70425SMarc-André Lureau }
9721e70425SMarc-André Lureau 
9821e70425SMarc-André Lureau static int vhost_kernel_set_vring_addr(struct vhost_dev *dev,
9921e70425SMarc-André Lureau                                        struct vhost_vring_addr *addr)
10021e70425SMarc-André Lureau {
10121e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_ADDR, addr);
10221e70425SMarc-André Lureau }
10321e70425SMarc-André Lureau 
10421e70425SMarc-André Lureau static int vhost_kernel_set_vring_endian(struct vhost_dev *dev,
10521e70425SMarc-André Lureau                                          struct vhost_vring_state *ring)
10621e70425SMarc-André Lureau {
10721e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_ENDIAN, ring);
10821e70425SMarc-André Lureau }
10921e70425SMarc-André Lureau 
11021e70425SMarc-André Lureau static int vhost_kernel_set_vring_num(struct vhost_dev *dev,
11121e70425SMarc-André Lureau                                       struct vhost_vring_state *ring)
11221e70425SMarc-André Lureau {
11321e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_NUM, ring);
11421e70425SMarc-André Lureau }
11521e70425SMarc-André Lureau 
11621e70425SMarc-André Lureau static int vhost_kernel_set_vring_base(struct vhost_dev *dev,
11721e70425SMarc-André Lureau                                        struct vhost_vring_state *ring)
11821e70425SMarc-André Lureau {
11921e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_BASE, ring);
12021e70425SMarc-André Lureau }
12121e70425SMarc-André Lureau 
12221e70425SMarc-André Lureau static int vhost_kernel_get_vring_base(struct vhost_dev *dev,
12321e70425SMarc-André Lureau                                        struct vhost_vring_state *ring)
12421e70425SMarc-André Lureau {
12521e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_GET_VRING_BASE, ring);
12621e70425SMarc-André Lureau }
12721e70425SMarc-André Lureau 
12821e70425SMarc-André Lureau static int vhost_kernel_set_vring_kick(struct vhost_dev *dev,
12921e70425SMarc-André Lureau                                        struct vhost_vring_file *file)
13021e70425SMarc-André Lureau {
13121e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_KICK, file);
13221e70425SMarc-André Lureau }
13321e70425SMarc-André Lureau 
13421e70425SMarc-André Lureau static int vhost_kernel_set_vring_call(struct vhost_dev *dev,
13521e70425SMarc-André Lureau                                        struct vhost_vring_file *file)
13621e70425SMarc-André Lureau {
13721e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_VRING_CALL, file);
13821e70425SMarc-André Lureau }
13921e70425SMarc-André Lureau 
14069e87b32SJason Wang static int vhost_kernel_set_vring_busyloop_timeout(struct vhost_dev *dev,
14169e87b32SJason Wang                                                    struct vhost_vring_state *s)
14269e87b32SJason Wang {
14369e87b32SJason Wang     return vhost_kernel_call(dev, VHOST_SET_VRING_BUSYLOOP_TIMEOUT, s);
14469e87b32SJason Wang }
14569e87b32SJason Wang 
14621e70425SMarc-André Lureau static int vhost_kernel_set_features(struct vhost_dev *dev,
14721e70425SMarc-André Lureau                                      uint64_t features)
14821e70425SMarc-André Lureau {
14921e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_FEATURES, &features);
15021e70425SMarc-André Lureau }
15121e70425SMarc-André Lureau 
15221e70425SMarc-André Lureau static int vhost_kernel_get_features(struct vhost_dev *dev,
15321e70425SMarc-André Lureau                                      uint64_t *features)
15421e70425SMarc-André Lureau {
15521e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_GET_FEATURES, features);
15621e70425SMarc-André Lureau }
15721e70425SMarc-André Lureau 
15821e70425SMarc-André Lureau static int vhost_kernel_set_owner(struct vhost_dev *dev)
15921e70425SMarc-André Lureau {
16021e70425SMarc-André Lureau     return vhost_kernel_call(dev, VHOST_SET_OWNER, NULL);
16121e70425SMarc-André Lureau }
16221e70425SMarc-André Lureau 
16321e70425SMarc-André Lureau static int vhost_kernel_reset_device(struct vhost_dev *dev)
16421e70425SMarc-André Lureau {
16560915dc4SYuanhan Liu     return vhost_kernel_call(dev, VHOST_RESET_OWNER, NULL);
16621e70425SMarc-André Lureau }
16721e70425SMarc-André Lureau 
16821e70425SMarc-André Lureau static int vhost_kernel_get_vq_index(struct vhost_dev *dev, int idx)
16921e70425SMarc-André Lureau {
17021e70425SMarc-André Lureau     assert(idx >= dev->vq_index && idx < dev->vq_index + dev->nvqs);
17121e70425SMarc-André Lureau 
17221e70425SMarc-André Lureau     return idx - dev->vq_index;
17321e70425SMarc-André Lureau }
17421e70425SMarc-André Lureau 
175fc0b9b0eSStefan Hajnoczi #ifdef CONFIG_VHOST_VSOCK
176fc0b9b0eSStefan Hajnoczi static int vhost_kernel_vsock_set_guest_cid(struct vhost_dev *dev,
177fc0b9b0eSStefan Hajnoczi                                             uint64_t guest_cid)
178fc0b9b0eSStefan Hajnoczi {
179fc0b9b0eSStefan Hajnoczi     return vhost_kernel_call(dev, VHOST_VSOCK_SET_GUEST_CID, &guest_cid);
180fc0b9b0eSStefan Hajnoczi }
181fc0b9b0eSStefan Hajnoczi 
182fc0b9b0eSStefan Hajnoczi static int vhost_kernel_vsock_set_running(struct vhost_dev *dev, int start)
183fc0b9b0eSStefan Hajnoczi {
184fc0b9b0eSStefan Hajnoczi     return vhost_kernel_call(dev, VHOST_VSOCK_SET_RUNNING, &start);
185fc0b9b0eSStefan Hajnoczi }
186fc0b9b0eSStefan Hajnoczi #endif /* CONFIG_VHOST_VSOCK */
187fc0b9b0eSStefan Hajnoczi 
188*c471ad0eSJason Wang static void vhost_kernel_iotlb_read(void *opaque)
189*c471ad0eSJason Wang {
190*c471ad0eSJason Wang     struct vhost_dev *dev = opaque;
191*c471ad0eSJason Wang     struct vhost_msg msg;
192*c471ad0eSJason Wang     ssize_t len;
193*c471ad0eSJason Wang 
194*c471ad0eSJason Wang     while ((len = read((uintptr_t)dev->opaque, &msg, sizeof msg)) > 0) {
195*c471ad0eSJason Wang         struct vhost_iotlb_msg *imsg = &msg.iotlb;
196*c471ad0eSJason Wang         if (len < sizeof msg) {
197*c471ad0eSJason Wang             error_report("Wrong vhost message len: %d", (int)len);
198*c471ad0eSJason Wang             break;
199*c471ad0eSJason Wang         }
200*c471ad0eSJason Wang         if (msg.type != VHOST_IOTLB_MSG) {
201*c471ad0eSJason Wang             error_report("Unknown vhost iotlb message type");
202*c471ad0eSJason Wang             break;
203*c471ad0eSJason Wang         }
204*c471ad0eSJason Wang         switch (imsg->type) {
205*c471ad0eSJason Wang         case VHOST_IOTLB_MISS:
206*c471ad0eSJason Wang             vhost_device_iotlb_miss(dev, imsg->iova,
207*c471ad0eSJason Wang                                     imsg->perm != VHOST_ACCESS_RO);
208*c471ad0eSJason Wang             break;
209*c471ad0eSJason Wang         case VHOST_IOTLB_UPDATE:
210*c471ad0eSJason Wang         case VHOST_IOTLB_INVALIDATE:
211*c471ad0eSJason Wang             error_report("Unexpected IOTLB message type");
212*c471ad0eSJason Wang             break;
213*c471ad0eSJason Wang         case VHOST_IOTLB_ACCESS_FAIL:
214*c471ad0eSJason Wang             /* FIXME: report device iotlb error */
215*c471ad0eSJason Wang             break;
216*c471ad0eSJason Wang         default:
217*c471ad0eSJason Wang             break;
218*c471ad0eSJason Wang         }
219*c471ad0eSJason Wang     }
220*c471ad0eSJason Wang }
221*c471ad0eSJason Wang 
222*c471ad0eSJason Wang static int vhost_kernel_update_device_iotlb(struct vhost_dev *dev,
223*c471ad0eSJason Wang                                             uint64_t iova, uint64_t uaddr,
224*c471ad0eSJason Wang                                             uint64_t len,
225*c471ad0eSJason Wang                                             IOMMUAccessFlags perm)
226*c471ad0eSJason Wang {
227*c471ad0eSJason Wang     struct vhost_msg msg;
228*c471ad0eSJason Wang     msg.type = VHOST_IOTLB_MSG;
229*c471ad0eSJason Wang     msg.iotlb.iova =  iova;
230*c471ad0eSJason Wang     msg.iotlb.uaddr = uaddr;
231*c471ad0eSJason Wang     msg.iotlb.size = len;
232*c471ad0eSJason Wang     msg.iotlb.type = VHOST_IOTLB_UPDATE;
233*c471ad0eSJason Wang 
234*c471ad0eSJason Wang     switch (perm) {
235*c471ad0eSJason Wang     case IOMMU_RO:
236*c471ad0eSJason Wang         msg.iotlb.perm = VHOST_ACCESS_RO;
237*c471ad0eSJason Wang         break;
238*c471ad0eSJason Wang     case IOMMU_WO:
239*c471ad0eSJason Wang         msg.iotlb.perm = VHOST_ACCESS_WO;
240*c471ad0eSJason Wang         break;
241*c471ad0eSJason Wang     case IOMMU_RW:
242*c471ad0eSJason Wang         msg.iotlb.perm = VHOST_ACCESS_RW;
243*c471ad0eSJason Wang         break;
244*c471ad0eSJason Wang     default:
245*c471ad0eSJason Wang         g_assert_not_reached();
246*c471ad0eSJason Wang     }
247*c471ad0eSJason Wang 
248*c471ad0eSJason Wang     if (write((uintptr_t)dev->opaque, &msg, sizeof msg) != sizeof msg) {
249*c471ad0eSJason Wang         error_report("Fail to update device iotlb");
250*c471ad0eSJason Wang         return -EFAULT;
251*c471ad0eSJason Wang     }
252*c471ad0eSJason Wang 
253*c471ad0eSJason Wang     return 0;
254*c471ad0eSJason Wang }
255*c471ad0eSJason Wang 
256*c471ad0eSJason Wang static int vhost_kernel_invalidate_device_iotlb(struct vhost_dev *dev,
257*c471ad0eSJason Wang                                                 uint64_t iova, uint64_t len)
258*c471ad0eSJason Wang {
259*c471ad0eSJason Wang     struct vhost_msg msg;
260*c471ad0eSJason Wang 
261*c471ad0eSJason Wang     msg.type = VHOST_IOTLB_MSG;
262*c471ad0eSJason Wang     msg.iotlb.iova = iova;
263*c471ad0eSJason Wang     msg.iotlb.size = len;
264*c471ad0eSJason Wang     msg.iotlb.type = VHOST_IOTLB_INVALIDATE;
265*c471ad0eSJason Wang 
266*c471ad0eSJason Wang     if (write((uintptr_t)dev->opaque, &msg, sizeof msg) != sizeof msg) {
267*c471ad0eSJason Wang         error_report("Fail to invalidate device iotlb");
268*c471ad0eSJason Wang         return -EFAULT;
269*c471ad0eSJason Wang     }
270*c471ad0eSJason Wang 
271*c471ad0eSJason Wang     return 0;
272*c471ad0eSJason Wang }
273*c471ad0eSJason Wang 
274*c471ad0eSJason Wang static void vhost_kernel_set_iotlb_callback(struct vhost_dev *dev,
275*c471ad0eSJason Wang                                            int enabled)
276*c471ad0eSJason Wang {
277*c471ad0eSJason Wang     if (enabled)
278*c471ad0eSJason Wang         qemu_set_fd_handler((uintptr_t)dev->opaque,
279*c471ad0eSJason Wang                             vhost_kernel_iotlb_read, NULL, dev);
280*c471ad0eSJason Wang     else
281*c471ad0eSJason Wang         qemu_set_fd_handler((uintptr_t)dev->opaque, NULL, NULL, NULL);
282*c471ad0eSJason Wang }
283*c471ad0eSJason Wang 
2841a1bfac9SNikolay Nikolaev static const VhostOps kernel_ops = {
2851a1bfac9SNikolay Nikolaev         .backend_type = VHOST_BACKEND_TYPE_KERNEL,
2861a1bfac9SNikolay Nikolaev         .vhost_backend_init = vhost_kernel_init,
287fc57fd99SYuanhan Liu         .vhost_backend_cleanup = vhost_kernel_cleanup,
2882ce68e4cSIgor Mammedov         .vhost_backend_memslots_limit = vhost_kernel_memslots_limit,
28921e70425SMarc-André Lureau         .vhost_net_set_backend = vhost_kernel_net_set_backend,
29021e70425SMarc-André Lureau         .vhost_scsi_set_endpoint = vhost_kernel_scsi_set_endpoint,
29121e70425SMarc-André Lureau         .vhost_scsi_clear_endpoint = vhost_kernel_scsi_clear_endpoint,
29221e70425SMarc-André Lureau         .vhost_scsi_get_abi_version = vhost_kernel_scsi_get_abi_version,
29321e70425SMarc-André Lureau         .vhost_set_log_base = vhost_kernel_set_log_base,
29421e70425SMarc-André Lureau         .vhost_set_mem_table = vhost_kernel_set_mem_table,
29521e70425SMarc-André Lureau         .vhost_set_vring_addr = vhost_kernel_set_vring_addr,
29621e70425SMarc-André Lureau         .vhost_set_vring_endian = vhost_kernel_set_vring_endian,
29721e70425SMarc-André Lureau         .vhost_set_vring_num = vhost_kernel_set_vring_num,
29821e70425SMarc-André Lureau         .vhost_set_vring_base = vhost_kernel_set_vring_base,
29921e70425SMarc-André Lureau         .vhost_get_vring_base = vhost_kernel_get_vring_base,
30021e70425SMarc-André Lureau         .vhost_set_vring_kick = vhost_kernel_set_vring_kick,
30121e70425SMarc-André Lureau         .vhost_set_vring_call = vhost_kernel_set_vring_call,
30269e87b32SJason Wang         .vhost_set_vring_busyloop_timeout =
30369e87b32SJason Wang                                 vhost_kernel_set_vring_busyloop_timeout,
30421e70425SMarc-André Lureau         .vhost_set_features = vhost_kernel_set_features,
30521e70425SMarc-André Lureau         .vhost_get_features = vhost_kernel_get_features,
30621e70425SMarc-André Lureau         .vhost_set_owner = vhost_kernel_set_owner,
30721e70425SMarc-André Lureau         .vhost_reset_device = vhost_kernel_reset_device,
30821e70425SMarc-André Lureau         .vhost_get_vq_index = vhost_kernel_get_vq_index,
309fc0b9b0eSStefan Hajnoczi #ifdef CONFIG_VHOST_VSOCK
310fc0b9b0eSStefan Hajnoczi         .vhost_vsock_set_guest_cid = vhost_kernel_vsock_set_guest_cid,
311fc0b9b0eSStefan Hajnoczi         .vhost_vsock_set_running = vhost_kernel_vsock_set_running,
312fc0b9b0eSStefan Hajnoczi #endif /* CONFIG_VHOST_VSOCK */
313*c471ad0eSJason Wang         .vhost_set_iotlb_callback = vhost_kernel_set_iotlb_callback,
314*c471ad0eSJason Wang         .vhost_update_device_iotlb = vhost_kernel_update_device_iotlb,
315*c471ad0eSJason Wang         .vhost_invalidate_device_iotlb = vhost_kernel_invalidate_device_iotlb,
3161a1bfac9SNikolay Nikolaev };
3171a1bfac9SNikolay Nikolaev 
3181a1bfac9SNikolay Nikolaev int vhost_set_backend_type(struct vhost_dev *dev, VhostBackendType backend_type)
3191a1bfac9SNikolay Nikolaev {
3201a1bfac9SNikolay Nikolaev     int r = 0;
3211a1bfac9SNikolay Nikolaev 
3221a1bfac9SNikolay Nikolaev     switch (backend_type) {
3231a1bfac9SNikolay Nikolaev     case VHOST_BACKEND_TYPE_KERNEL:
3241a1bfac9SNikolay Nikolaev         dev->vhost_ops = &kernel_ops;
3251a1bfac9SNikolay Nikolaev         break;
3265f6f6664SNikolay Nikolaev     case VHOST_BACKEND_TYPE_USER:
3275f6f6664SNikolay Nikolaev         dev->vhost_ops = &user_ops;
3285f6f6664SNikolay Nikolaev         break;
3291a1bfac9SNikolay Nikolaev     default:
330c6547661SGonglei         error_report("Unknown vhost backend type");
3311a1bfac9SNikolay Nikolaev         r = -1;
3321a1bfac9SNikolay Nikolaev     }
3331a1bfac9SNikolay Nikolaev 
3341a1bfac9SNikolay Nikolaev     return r;
3351a1bfac9SNikolay Nikolaev }
336