Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-or-later
2 : /* Driver for Virtio crypto device.
3 : *
4 : * Copyright 2016 HUAWEI TECHNOLOGIES CO., LTD.
5 : */
6 :
7 : #include <linux/err.h>
8 : #include <linux/module.h>
9 : #include <linux/virtio_config.h>
10 : #include <linux/cpu.h>
11 :
12 : #include <uapi/linux/virtio_crypto.h>
13 : #include "virtio_crypto_common.h"
14 :
15 :
16 : void
17 0 : virtcrypto_clear_request(struct virtio_crypto_request *vc_req)
18 : {
19 0 : if (vc_req) {
20 0 : kfree_sensitive(vc_req->req_data);
21 0 : kfree(vc_req->sgs);
22 : }
23 0 : }
24 :
25 0 : static void virtcrypto_dataq_callback(struct virtqueue *vq)
26 : {
27 0 : struct virtio_crypto *vcrypto = vq->vdev->priv;
28 0 : struct virtio_crypto_request *vc_req;
29 0 : unsigned long flags;
30 0 : unsigned int len;
31 0 : unsigned int qid = vq->index;
32 :
33 0 : spin_lock_irqsave(&vcrypto->data_vq[qid].lock, flags);
34 0 : do {
35 0 : virtqueue_disable_cb(vq);
36 0 : while ((vc_req = virtqueue_get_buf(vq, &len)) != NULL) {
37 0 : spin_unlock_irqrestore(
38 0 : &vcrypto->data_vq[qid].lock, flags);
39 0 : if (vc_req->alg_cb)
40 0 : vc_req->alg_cb(vc_req, len);
41 0 : spin_lock_irqsave(
42 : &vcrypto->data_vq[qid].lock, flags);
43 : }
44 0 : } while (!virtqueue_enable_cb(vq));
45 0 : spin_unlock_irqrestore(&vcrypto->data_vq[qid].lock, flags);
46 0 : }
47 :
48 0 : static int virtcrypto_find_vqs(struct virtio_crypto *vi)
49 : {
50 0 : vq_callback_t **callbacks;
51 0 : struct virtqueue **vqs;
52 0 : int ret = -ENOMEM;
53 0 : int i, total_vqs;
54 0 : const char **names;
55 0 : struct device *dev = &vi->vdev->dev;
56 :
57 : /*
58 : * We expect 1 data virtqueue, followed by
59 : * possible N-1 data queues used in multiqueue mode,
60 : * followed by control vq.
61 : */
62 0 : total_vqs = vi->max_data_queues + 1;
63 :
64 : /* Allocate space for find_vqs parameters */
65 0 : vqs = kcalloc(total_vqs, sizeof(*vqs), GFP_KERNEL);
66 0 : if (!vqs)
67 0 : goto err_vq;
68 0 : callbacks = kcalloc(total_vqs, sizeof(*callbacks), GFP_KERNEL);
69 0 : if (!callbacks)
70 0 : goto err_callback;
71 0 : names = kcalloc(total_vqs, sizeof(*names), GFP_KERNEL);
72 0 : if (!names)
73 0 : goto err_names;
74 :
75 : /* Parameters for control virtqueue */
76 0 : callbacks[total_vqs - 1] = NULL;
77 0 : names[total_vqs - 1] = "controlq";
78 :
79 : /* Allocate/initialize parameters for data virtqueues */
80 0 : for (i = 0; i < vi->max_data_queues; i++) {
81 0 : callbacks[i] = virtcrypto_dataq_callback;
82 0 : snprintf(vi->data_vq[i].name, sizeof(vi->data_vq[i].name),
83 : "dataq.%d", i);
84 0 : names[i] = vi->data_vq[i].name;
85 : }
86 :
87 0 : ret = virtio_find_vqs(vi->vdev, total_vqs, vqs, callbacks, names, NULL);
88 0 : if (ret)
89 0 : goto err_find;
90 :
91 0 : vi->ctrl_vq = vqs[total_vqs - 1];
92 :
93 0 : for (i = 0; i < vi->max_data_queues; i++) {
94 0 : spin_lock_init(&vi->data_vq[i].lock);
95 0 : vi->data_vq[i].vq = vqs[i];
96 : /* Initialize crypto engine */
97 0 : vi->data_vq[i].engine = crypto_engine_alloc_init(dev, 1);
98 0 : if (!vi->data_vq[i].engine) {
99 0 : ret = -ENOMEM;
100 0 : goto err_engine;
101 : }
102 : }
103 :
104 0 : kfree(names);
105 0 : kfree(callbacks);
106 0 : kfree(vqs);
107 :
108 0 : return 0;
109 :
110 0 : err_engine:
111 0 : err_find:
112 0 : kfree(names);
113 0 : err_names:
114 0 : kfree(callbacks);
115 0 : err_callback:
116 0 : kfree(vqs);
117 : err_vq:
118 : return ret;
119 : }
120 :
121 0 : static int virtcrypto_alloc_queues(struct virtio_crypto *vi)
122 : {
123 0 : vi->data_vq = kcalloc(vi->max_data_queues, sizeof(*vi->data_vq),
124 : GFP_KERNEL);
125 0 : if (!vi->data_vq)
126 0 : return -ENOMEM;
127 :
128 : return 0;
129 : }
130 :
131 0 : static void virtcrypto_clean_affinity(struct virtio_crypto *vi, long hcpu)
132 : {
133 0 : int i;
134 :
135 0 : if (vi->affinity_hint_set) {
136 0 : for (i = 0; i < vi->max_data_queues; i++)
137 0 : virtqueue_set_affinity(vi->data_vq[i].vq, NULL);
138 :
139 0 : vi->affinity_hint_set = false;
140 : }
141 0 : }
142 :
143 0 : static void virtcrypto_set_affinity(struct virtio_crypto *vcrypto)
144 : {
145 0 : int i = 0;
146 0 : int cpu;
147 :
148 : /*
149 : * In single queue mode, we don't set the cpu affinity.
150 : */
151 0 : if (vcrypto->curr_queue == 1 || vcrypto->max_data_queues == 1) {
152 0 : virtcrypto_clean_affinity(vcrypto, -1);
153 0 : return;
154 : }
155 :
156 : /*
157 : * In multiqueue mode, we let the queue to be private to one cpu
158 : * by setting the affinity hint to eliminate the contention.
159 : *
160 : * TODO: adds cpu hotplug support by register cpu notifier.
161 : *
162 : */
163 0 : for_each_online_cpu(cpu) {
164 0 : virtqueue_set_affinity(vcrypto->data_vq[i].vq, cpumask_of(cpu));
165 0 : if (++i >= vcrypto->max_data_queues)
166 : break;
167 : }
168 :
169 0 : vcrypto->affinity_hint_set = true;
170 : }
171 :
172 0 : static void virtcrypto_free_queues(struct virtio_crypto *vi)
173 : {
174 0 : kfree(vi->data_vq);
175 0 : }
176 :
177 0 : static int virtcrypto_init_vqs(struct virtio_crypto *vi)
178 : {
179 0 : int ret;
180 :
181 : /* Allocate send & receive queues */
182 0 : ret = virtcrypto_alloc_queues(vi);
183 0 : if (ret)
184 0 : goto err;
185 :
186 0 : ret = virtcrypto_find_vqs(vi);
187 0 : if (ret)
188 0 : goto err_free;
189 :
190 0 : get_online_cpus();
191 0 : virtcrypto_set_affinity(vi);
192 0 : put_online_cpus();
193 :
194 0 : return 0;
195 :
196 0 : err_free:
197 0 : virtcrypto_free_queues(vi);
198 : err:
199 : return ret;
200 : }
201 :
202 0 : static int virtcrypto_update_status(struct virtio_crypto *vcrypto)
203 : {
204 0 : u32 status;
205 0 : int err;
206 :
207 0 : virtio_cread_le(vcrypto->vdev,
208 : struct virtio_crypto_config, status, &status);
209 :
210 : /*
211 : * Unknown status bits would be a host error and the driver
212 : * should consider the device to be broken.
213 : */
214 0 : if (status & (~VIRTIO_CRYPTO_S_HW_READY)) {
215 0 : dev_warn(&vcrypto->vdev->dev,
216 : "Unknown status bits: 0x%x\n", status);
217 :
218 0 : virtio_break_device(vcrypto->vdev);
219 0 : return -EPERM;
220 : }
221 :
222 0 : if (vcrypto->status == status)
223 : return 0;
224 :
225 0 : vcrypto->status = status;
226 :
227 0 : if (vcrypto->status & VIRTIO_CRYPTO_S_HW_READY) {
228 0 : err = virtcrypto_dev_start(vcrypto);
229 0 : if (err) {
230 0 : dev_err(&vcrypto->vdev->dev,
231 : "Failed to start virtio crypto device.\n");
232 :
233 0 : return -EPERM;
234 : }
235 0 : dev_info(&vcrypto->vdev->dev, "Accelerator device is ready\n");
236 : } else {
237 0 : virtcrypto_dev_stop(vcrypto);
238 0 : dev_info(&vcrypto->vdev->dev, "Accelerator is not ready\n");
239 : }
240 :
241 : return 0;
242 : }
243 :
244 0 : static int virtcrypto_start_crypto_engines(struct virtio_crypto *vcrypto)
245 : {
246 0 : int32_t i;
247 0 : int ret;
248 :
249 0 : for (i = 0; i < vcrypto->max_data_queues; i++) {
250 0 : if (vcrypto->data_vq[i].engine) {
251 0 : ret = crypto_engine_start(vcrypto->data_vq[i].engine);
252 0 : if (ret)
253 0 : goto err;
254 : }
255 : }
256 :
257 : return 0;
258 :
259 0 : err:
260 0 : while (--i >= 0)
261 0 : if (vcrypto->data_vq[i].engine)
262 0 : crypto_engine_exit(vcrypto->data_vq[i].engine);
263 :
264 : return ret;
265 : }
266 :
267 0 : static void virtcrypto_clear_crypto_engines(struct virtio_crypto *vcrypto)
268 : {
269 0 : u32 i;
270 :
271 0 : for (i = 0; i < vcrypto->max_data_queues; i++)
272 0 : if (vcrypto->data_vq[i].engine)
273 0 : crypto_engine_exit(vcrypto->data_vq[i].engine);
274 0 : }
275 :
276 0 : static void virtcrypto_del_vqs(struct virtio_crypto *vcrypto)
277 : {
278 0 : struct virtio_device *vdev = vcrypto->vdev;
279 :
280 0 : virtcrypto_clean_affinity(vcrypto, -1);
281 :
282 0 : vdev->config->del_vqs(vdev);
283 :
284 0 : virtcrypto_free_queues(vcrypto);
285 0 : }
286 :
287 0 : static int virtcrypto_probe(struct virtio_device *vdev)
288 : {
289 0 : int err = -EFAULT;
290 0 : struct virtio_crypto *vcrypto;
291 0 : u32 max_data_queues = 0, max_cipher_key_len = 0;
292 0 : u32 max_auth_key_len = 0;
293 0 : u64 max_size = 0;
294 0 : u32 cipher_algo_l = 0;
295 0 : u32 cipher_algo_h = 0;
296 0 : u32 hash_algo = 0;
297 0 : u32 mac_algo_l = 0;
298 0 : u32 mac_algo_h = 0;
299 0 : u32 aead_algo = 0;
300 0 : u32 crypto_services = 0;
301 :
302 0 : if (!virtio_has_feature(vdev, VIRTIO_F_VERSION_1))
303 : return -ENODEV;
304 :
305 0 : if (!vdev->config->get) {
306 0 : dev_err(&vdev->dev, "%s failure: config access disabled\n",
307 : __func__);
308 0 : return -EINVAL;
309 : }
310 :
311 0 : if (num_possible_nodes() > 1 && dev_to_node(&vdev->dev) < 0) {
312 : /*
313 : * If the accelerator is connected to a node with no memory
314 : * there is no point in using the accelerator since the remote
315 : * memory transaction will be very slow.
316 : */
317 0 : dev_err(&vdev->dev, "Invalid NUMA configuration.\n");
318 0 : return -EINVAL;
319 : }
320 :
321 0 : vcrypto = kzalloc_node(sizeof(*vcrypto), GFP_KERNEL,
322 : dev_to_node(&vdev->dev));
323 0 : if (!vcrypto)
324 : return -ENOMEM;
325 :
326 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
327 : max_dataqueues, &max_data_queues);
328 0 : if (max_data_queues < 1)
329 : max_data_queues = 1;
330 :
331 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
332 : max_cipher_key_len, &max_cipher_key_len);
333 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
334 : max_auth_key_len, &max_auth_key_len);
335 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
336 : max_size, &max_size);
337 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
338 : crypto_services, &crypto_services);
339 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
340 : cipher_algo_l, &cipher_algo_l);
341 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
342 : cipher_algo_h, &cipher_algo_h);
343 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
344 : hash_algo, &hash_algo);
345 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
346 : mac_algo_l, &mac_algo_l);
347 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
348 : mac_algo_h, &mac_algo_h);
349 0 : virtio_cread_le(vdev, struct virtio_crypto_config,
350 : aead_algo, &aead_algo);
351 :
352 : /* Add virtio crypto device to global table */
353 0 : err = virtcrypto_devmgr_add_dev(vcrypto);
354 0 : if (err) {
355 0 : dev_err(&vdev->dev, "Failed to add new virtio crypto device.\n");
356 0 : goto free;
357 : }
358 0 : vcrypto->owner = THIS_MODULE;
359 0 : vcrypto = vdev->priv = vcrypto;
360 0 : vcrypto->vdev = vdev;
361 :
362 0 : spin_lock_init(&vcrypto->ctrl_lock);
363 :
364 : /* Use single data queue as default */
365 0 : vcrypto->curr_queue = 1;
366 0 : vcrypto->max_data_queues = max_data_queues;
367 0 : vcrypto->max_cipher_key_len = max_cipher_key_len;
368 0 : vcrypto->max_auth_key_len = max_auth_key_len;
369 0 : vcrypto->max_size = max_size;
370 0 : vcrypto->crypto_services = crypto_services;
371 0 : vcrypto->cipher_algo_l = cipher_algo_l;
372 0 : vcrypto->cipher_algo_h = cipher_algo_h;
373 0 : vcrypto->mac_algo_l = mac_algo_l;
374 0 : vcrypto->mac_algo_h = mac_algo_h;
375 0 : vcrypto->hash_algo = hash_algo;
376 0 : vcrypto->aead_algo = aead_algo;
377 :
378 :
379 0 : dev_info(&vdev->dev,
380 : "max_queues: %u, max_cipher_key_len: %u, max_auth_key_len: %u, max_size 0x%llx\n",
381 : vcrypto->max_data_queues,
382 : vcrypto->max_cipher_key_len,
383 : vcrypto->max_auth_key_len,
384 : vcrypto->max_size);
385 :
386 0 : err = virtcrypto_init_vqs(vcrypto);
387 0 : if (err) {
388 0 : dev_err(&vdev->dev, "Failed to initialize vqs.\n");
389 0 : goto free_dev;
390 : }
391 :
392 0 : err = virtcrypto_start_crypto_engines(vcrypto);
393 0 : if (err)
394 0 : goto free_vqs;
395 :
396 0 : virtio_device_ready(vdev);
397 :
398 0 : err = virtcrypto_update_status(vcrypto);
399 0 : if (err)
400 0 : goto free_engines;
401 :
402 : return 0;
403 :
404 0 : free_engines:
405 0 : virtcrypto_clear_crypto_engines(vcrypto);
406 0 : free_vqs:
407 0 : vcrypto->vdev->config->reset(vdev);
408 0 : virtcrypto_del_vqs(vcrypto);
409 0 : free_dev:
410 0 : virtcrypto_devmgr_rm_dev(vcrypto);
411 0 : free:
412 0 : kfree(vcrypto);
413 0 : return err;
414 : }
415 :
416 0 : static void virtcrypto_free_unused_reqs(struct virtio_crypto *vcrypto)
417 : {
418 0 : struct virtio_crypto_request *vc_req;
419 0 : int i;
420 0 : struct virtqueue *vq;
421 :
422 0 : for (i = 0; i < vcrypto->max_data_queues; i++) {
423 0 : vq = vcrypto->data_vq[i].vq;
424 0 : while ((vc_req = virtqueue_detach_unused_buf(vq)) != NULL) {
425 0 : kfree(vc_req->req_data);
426 0 : kfree(vc_req->sgs);
427 : }
428 : }
429 0 : }
430 :
431 0 : static void virtcrypto_remove(struct virtio_device *vdev)
432 : {
433 0 : struct virtio_crypto *vcrypto = vdev->priv;
434 :
435 0 : dev_info(&vdev->dev, "Start virtcrypto_remove.\n");
436 :
437 0 : if (virtcrypto_dev_started(vcrypto))
438 0 : virtcrypto_dev_stop(vcrypto);
439 0 : vdev->config->reset(vdev);
440 0 : virtcrypto_free_unused_reqs(vcrypto);
441 0 : virtcrypto_clear_crypto_engines(vcrypto);
442 0 : virtcrypto_del_vqs(vcrypto);
443 0 : virtcrypto_devmgr_rm_dev(vcrypto);
444 0 : kfree(vcrypto);
445 0 : }
446 :
447 0 : static void virtcrypto_config_changed(struct virtio_device *vdev)
448 : {
449 0 : struct virtio_crypto *vcrypto = vdev->priv;
450 :
451 0 : virtcrypto_update_status(vcrypto);
452 0 : }
453 :
454 : #ifdef CONFIG_PM_SLEEP
455 : static int virtcrypto_freeze(struct virtio_device *vdev)
456 : {
457 : struct virtio_crypto *vcrypto = vdev->priv;
458 :
459 : vdev->config->reset(vdev);
460 : virtcrypto_free_unused_reqs(vcrypto);
461 : if (virtcrypto_dev_started(vcrypto))
462 : virtcrypto_dev_stop(vcrypto);
463 :
464 : virtcrypto_clear_crypto_engines(vcrypto);
465 : virtcrypto_del_vqs(vcrypto);
466 : return 0;
467 : }
468 :
469 : static int virtcrypto_restore(struct virtio_device *vdev)
470 : {
471 : struct virtio_crypto *vcrypto = vdev->priv;
472 : int err;
473 :
474 : err = virtcrypto_init_vqs(vcrypto);
475 : if (err)
476 : return err;
477 :
478 : err = virtcrypto_start_crypto_engines(vcrypto);
479 : if (err)
480 : goto free_vqs;
481 :
482 : virtio_device_ready(vdev);
483 :
484 : err = virtcrypto_dev_start(vcrypto);
485 : if (err) {
486 : dev_err(&vdev->dev, "Failed to start virtio crypto device.\n");
487 : goto free_engines;
488 : }
489 :
490 : return 0;
491 :
492 : free_engines:
493 : virtcrypto_clear_crypto_engines(vcrypto);
494 : free_vqs:
495 : vcrypto->vdev->config->reset(vdev);
496 : virtcrypto_del_vqs(vcrypto);
497 : return err;
498 : }
499 : #endif
500 :
501 : static const unsigned int features[] = {
502 : /* none */
503 : };
504 :
505 : static const struct virtio_device_id id_table[] = {
506 : { VIRTIO_ID_CRYPTO, VIRTIO_DEV_ANY_ID },
507 : { 0 },
508 : };
509 :
510 : static struct virtio_driver virtio_crypto_driver = {
511 : .driver.name = KBUILD_MODNAME,
512 : .driver.owner = THIS_MODULE,
513 : .feature_table = features,
514 : .feature_table_size = ARRAY_SIZE(features),
515 : .id_table = id_table,
516 : .probe = virtcrypto_probe,
517 : .remove = virtcrypto_remove,
518 : .config_changed = virtcrypto_config_changed,
519 : #ifdef CONFIG_PM_SLEEP
520 : .freeze = virtcrypto_freeze,
521 : .restore = virtcrypto_restore,
522 : #endif
523 : };
524 :
525 1 : module_virtio_driver(virtio_crypto_driver);
526 :
527 : MODULE_DEVICE_TABLE(virtio, id_table);
528 : MODULE_DESCRIPTION("virtio crypto device driver");
529 : MODULE_LICENSE("GPL");
530 : MODULE_AUTHOR("Gonglei <arei.gonglei@huawei.com>");
|
