Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * Dynamic DMA mapping support.
4 : *
5 : * This implementation is a fallback for platforms that do not support
6 : * I/O TLBs (aka DMA address translation hardware).
7 : * Copyright (C) 2000 Asit Mallick <Asit.K.Mallick@intel.com>
8 : * Copyright (C) 2000 Goutham Rao <goutham.rao@intel.com>
9 : * Copyright (C) 2000, 2003 Hewlett-Packard Co
10 : * David Mosberger-Tang <davidm@hpl.hp.com>
11 : *
12 : * 03/05/07 davidm Switch from PCI-DMA to generic device DMA API.
13 : * 00/12/13 davidm Rename to swiotlb.c and add mark_clean() to avoid
14 : * unnecessary i-cache flushing.
15 : * 04/07/.. ak Better overflow handling. Assorted fixes.
16 : * 05/09/10 linville Add support for syncing ranges, support syncing for
17 : * DMA_BIDIRECTIONAL mappings, miscellaneous cleanup.
18 : * 08/12/11 beckyb Add highmem support
19 : */
20 :
21 : #define pr_fmt(fmt) "software IO TLB: " fmt
22 :
23 : #include <linux/cache.h>
24 : #include <linux/dma-direct.h>
25 : #include <linux/dma-map-ops.h>
26 : #include <linux/mm.h>
27 : #include <linux/export.h>
28 : #include <linux/spinlock.h>
29 : #include <linux/string.h>
30 : #include <linux/swiotlb.h>
31 : #include <linux/pfn.h>
32 : #include <linux/types.h>
33 : #include <linux/ctype.h>
34 : #include <linux/highmem.h>
35 : #include <linux/gfp.h>
36 : #include <linux/scatterlist.h>
37 : #include <linux/mem_encrypt.h>
38 : #include <linux/set_memory.h>
39 : #ifdef CONFIG_DEBUG_FS
40 : #include <linux/debugfs.h>
41 : #endif
42 :
43 : #include <asm/io.h>
44 : #include <asm/dma.h>
45 :
46 : #include <linux/init.h>
47 : #include <linux/memblock.h>
48 : #include <linux/iommu-helper.h>
49 :
50 : #define CREATE_TRACE_POINTS
51 : #include <trace/events/swiotlb.h>
52 :
53 : #define SLABS_PER_PAGE (1 << (PAGE_SHIFT - IO_TLB_SHIFT))
54 :
55 : /*
56 : * Minimum IO TLB size to bother booting with. Systems with mainly
57 : * 64bit capable cards will only lightly use the swiotlb. If we can't
58 : * allocate a contiguous 1MB, we're probably in trouble anyway.
59 : */
60 : #define IO_TLB_MIN_SLABS ((1<<20) >> IO_TLB_SHIFT)
61 :
62 : enum swiotlb_force swiotlb_force;
63 :
64 : /*
65 : * Used to do a quick range check in swiotlb_tbl_unmap_single and
66 : * swiotlb_tbl_sync_single_*, to see if the memory was in fact allocated by this
67 : * API.
68 : */
69 : phys_addr_t io_tlb_start, io_tlb_end;
70 :
71 : /*
72 : * The number of IO TLB blocks (in groups of 64) between io_tlb_start and
73 : * io_tlb_end. This is command line adjustable via setup_io_tlb_npages.
74 : */
75 : static unsigned long io_tlb_nslabs;
76 :
77 : /*
78 : * The number of used IO TLB block
79 : */
80 : static unsigned long io_tlb_used;
81 :
82 : /*
83 : * This is a free list describing the number of free entries available from
84 : * each index
85 : */
86 : static unsigned int *io_tlb_list;
87 : static unsigned int io_tlb_index;
88 :
89 : /*
90 : * Max segment that we can provide which (if pages are contingous) will
91 : * not be bounced (unless SWIOTLB_FORCE is set).
92 : */
93 : static unsigned int max_segment;
94 :
95 : /*
96 : * We need to save away the original address corresponding to a mapped entry
97 : * for the sync operations.
98 : */
99 : #define INVALID_PHYS_ADDR (~(phys_addr_t)0)
100 : static phys_addr_t *io_tlb_orig_addr;
101 :
102 : /*
103 : * The mapped buffer's size should be validated during a sync operation.
104 : */
105 : static size_t *io_tlb_orig_size;
106 :
107 : /*
108 : * Protect the above data structures in the map and unmap calls
109 : */
110 : static DEFINE_SPINLOCK(io_tlb_lock);
111 :
112 : static int late_alloc;
113 :
114 : static int __init
115 0 : setup_io_tlb_npages(char *str)
116 : {
117 0 : if (isdigit(*str)) {
118 0 : io_tlb_nslabs = simple_strtoul(str, &str, 0);
119 : /* avoid tail segment of size < IO_TLB_SEGSIZE */
120 0 : io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
121 : }
122 0 : if (*str == ',')
123 0 : ++str;
124 0 : if (!strcmp(str, "force")) {
125 0 : swiotlb_force = SWIOTLB_FORCE;
126 0 : } else if (!strcmp(str, "noforce")) {
127 0 : swiotlb_force = SWIOTLB_NO_FORCE;
128 0 : io_tlb_nslabs = 1;
129 : }
130 :
131 0 : return 0;
132 : }
133 : early_param("swiotlb", setup_io_tlb_npages);
134 :
135 : static bool no_iotlb_memory;
136 :
137 0 : unsigned long swiotlb_nr_tbl(void)
138 : {
139 0 : return unlikely(no_iotlb_memory) ? 0 : io_tlb_nslabs;
140 : }
141 : EXPORT_SYMBOL_GPL(swiotlb_nr_tbl);
142 :
143 0 : unsigned int swiotlb_max_segment(void)
144 : {
145 0 : return unlikely(no_iotlb_memory) ? 0 : max_segment;
146 : }
147 : EXPORT_SYMBOL_GPL(swiotlb_max_segment);
148 :
149 0 : void swiotlb_set_max_segment(unsigned int val)
150 : {
151 0 : if (swiotlb_force == SWIOTLB_FORCE)
152 0 : max_segment = 1;
153 : else
154 0 : max_segment = rounddown(val, PAGE_SIZE);
155 0 : }
156 :
157 0 : unsigned long swiotlb_size_or_default(void)
158 : {
159 0 : unsigned long size;
160 :
161 0 : size = io_tlb_nslabs << IO_TLB_SHIFT;
162 :
163 0 : return size ? size : (IO_TLB_DEFAULT_SIZE);
164 : }
165 :
166 0 : void __init swiotlb_adjust_size(unsigned long new_size)
167 : {
168 0 : unsigned long size;
169 :
170 : /*
171 : * If swiotlb parameter has not been specified, give a chance to
172 : * architectures such as those supporting memory encryption to
173 : * adjust/expand SWIOTLB size for their use.
174 : */
175 0 : if (!io_tlb_nslabs) {
176 0 : size = ALIGN(new_size, IO_TLB_SIZE);
177 0 : io_tlb_nslabs = size >> IO_TLB_SHIFT;
178 0 : io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
179 :
180 0 : pr_info("SWIOTLB bounce buffer size adjusted to %luMB", size >> 20);
181 : }
182 0 : }
183 :
184 0 : void swiotlb_print_info(void)
185 : {
186 0 : unsigned long bytes = io_tlb_nslabs << IO_TLB_SHIFT;
187 :
188 0 : if (no_iotlb_memory) {
189 0 : pr_warn("No low mem\n");
190 0 : return;
191 : }
192 :
193 0 : pr_info("mapped [mem %pa-%pa] (%luMB)\n", &io_tlb_start, &io_tlb_end,
194 : bytes >> 20);
195 : }
196 :
197 0 : static inline unsigned long io_tlb_offset(unsigned long val)
198 : {
199 0 : return val & (IO_TLB_SEGSIZE - 1);
200 : }
201 :
202 0 : static inline unsigned long nr_slots(u64 val)
203 : {
204 0 : return DIV_ROUND_UP(val, IO_TLB_SIZE);
205 : }
206 :
207 : /*
208 : * Early SWIOTLB allocation may be too early to allow an architecture to
209 : * perform the desired operations. This function allows the architecture to
210 : * call SWIOTLB when the operations are possible. It needs to be called
211 : * before the SWIOTLB memory is used.
212 : */
213 0 : void __init swiotlb_update_mem_attributes(void)
214 : {
215 0 : void *vaddr;
216 0 : unsigned long bytes;
217 :
218 0 : if (no_iotlb_memory || late_alloc)
219 : return;
220 :
221 0 : vaddr = phys_to_virt(io_tlb_start);
222 0 : bytes = PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT);
223 0 : set_memory_decrypted((unsigned long)vaddr, bytes >> PAGE_SHIFT);
224 0 : memset(vaddr, 0, bytes);
225 : }
226 :
227 0 : int __init swiotlb_init_with_tbl(char *tlb, unsigned long nslabs, int verbose)
228 : {
229 0 : unsigned long i, bytes;
230 0 : size_t alloc_size;
231 :
232 0 : bytes = nslabs << IO_TLB_SHIFT;
233 :
234 0 : io_tlb_nslabs = nslabs;
235 0 : io_tlb_start = __pa(tlb);
236 0 : io_tlb_end = io_tlb_start + bytes;
237 :
238 : /*
239 : * Allocate and initialize the free list array. This array is used
240 : * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
241 : * between io_tlb_start and io_tlb_end.
242 : */
243 0 : alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(int));
244 0 : io_tlb_list = memblock_alloc(alloc_size, PAGE_SIZE);
245 0 : if (!io_tlb_list)
246 0 : panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
247 : __func__, alloc_size, PAGE_SIZE);
248 :
249 0 : alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t));
250 0 : io_tlb_orig_addr = memblock_alloc(alloc_size, PAGE_SIZE);
251 0 : if (!io_tlb_orig_addr)
252 0 : panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
253 : __func__, alloc_size, PAGE_SIZE);
254 :
255 0 : alloc_size = PAGE_ALIGN(io_tlb_nslabs * sizeof(size_t));
256 0 : io_tlb_orig_size = memblock_alloc(alloc_size, PAGE_SIZE);
257 0 : if (!io_tlb_orig_size)
258 0 : panic("%s: Failed to allocate %zu bytes align=0x%lx\n",
259 : __func__, alloc_size, PAGE_SIZE);
260 :
261 0 : for (i = 0; i < io_tlb_nslabs; i++) {
262 0 : io_tlb_list[i] = IO_TLB_SEGSIZE - io_tlb_offset(i);
263 0 : io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
264 0 : io_tlb_orig_size[i] = 0;
265 : }
266 0 : io_tlb_index = 0;
267 0 : no_iotlb_memory = false;
268 :
269 0 : if (verbose)
270 0 : swiotlb_print_info();
271 :
272 0 : swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT);
273 0 : return 0;
274 : }
275 :
276 : /*
277 : * Statically reserve bounce buffer space and initialize bounce buffer data
278 : * structures for the software IO TLB used to implement the DMA API.
279 : */
280 : void __init
281 0 : swiotlb_init(int verbose)
282 : {
283 0 : size_t default_size = IO_TLB_DEFAULT_SIZE;
284 0 : unsigned char *vstart;
285 0 : unsigned long bytes;
286 :
287 0 : if (!io_tlb_nslabs) {
288 0 : io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
289 0 : io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
290 : }
291 :
292 0 : bytes = io_tlb_nslabs << IO_TLB_SHIFT;
293 :
294 : /* Get IO TLB memory from the low pages */
295 0 : vstart = memblock_alloc_low(PAGE_ALIGN(bytes), PAGE_SIZE);
296 0 : if (vstart && !swiotlb_init_with_tbl(vstart, io_tlb_nslabs, verbose))
297 : return;
298 :
299 0 : if (io_tlb_start) {
300 0 : memblock_free_early(io_tlb_start,
301 0 : PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
302 0 : io_tlb_start = 0;
303 : }
304 0 : pr_warn("Cannot allocate buffer");
305 0 : no_iotlb_memory = true;
306 : }
307 :
308 : /*
309 : * Systems with larger DMA zones (those that don't support ISA) can
310 : * initialize the swiotlb later using the slab allocator if needed.
311 : * This should be just like above, but with some error catching.
312 : */
313 : int
314 0 : swiotlb_late_init_with_default_size(size_t default_size)
315 : {
316 0 : unsigned long bytes, req_nslabs = io_tlb_nslabs;
317 0 : unsigned char *vstart = NULL;
318 0 : unsigned int order;
319 0 : int rc = 0;
320 :
321 0 : if (!io_tlb_nslabs) {
322 0 : io_tlb_nslabs = (default_size >> IO_TLB_SHIFT);
323 0 : io_tlb_nslabs = ALIGN(io_tlb_nslabs, IO_TLB_SEGSIZE);
324 : }
325 :
326 : /*
327 : * Get IO TLB memory from the low pages
328 : */
329 0 : order = get_order(io_tlb_nslabs << IO_TLB_SHIFT);
330 0 : io_tlb_nslabs = SLABS_PER_PAGE << order;
331 0 : bytes = io_tlb_nslabs << IO_TLB_SHIFT;
332 :
333 0 : while ((SLABS_PER_PAGE << order) > IO_TLB_MIN_SLABS) {
334 0 : vstart = (void *)__get_free_pages(GFP_DMA | __GFP_NOWARN,
335 : order);
336 0 : if (vstart)
337 : break;
338 0 : order--;
339 : }
340 :
341 0 : if (!vstart) {
342 0 : io_tlb_nslabs = req_nslabs;
343 0 : return -ENOMEM;
344 : }
345 0 : if (order != get_order(bytes)) {
346 0 : pr_warn("only able to allocate %ld MB\n",
347 : (PAGE_SIZE << order) >> 20);
348 0 : io_tlb_nslabs = SLABS_PER_PAGE << order;
349 : }
350 0 : rc = swiotlb_late_init_with_tbl(vstart, io_tlb_nslabs);
351 0 : if (rc)
352 0 : free_pages((unsigned long)vstart, order);
353 :
354 : return rc;
355 : }
356 :
357 0 : static void swiotlb_cleanup(void)
358 : {
359 0 : io_tlb_end = 0;
360 0 : io_tlb_start = 0;
361 0 : io_tlb_nslabs = 0;
362 0 : max_segment = 0;
363 0 : }
364 :
365 : int
366 0 : swiotlb_late_init_with_tbl(char *tlb, unsigned long nslabs)
367 : {
368 0 : unsigned long i, bytes;
369 :
370 0 : bytes = nslabs << IO_TLB_SHIFT;
371 :
372 0 : io_tlb_nslabs = nslabs;
373 0 : io_tlb_start = virt_to_phys(tlb);
374 0 : io_tlb_end = io_tlb_start + bytes;
375 :
376 0 : set_memory_decrypted((unsigned long)tlb, bytes >> PAGE_SHIFT);
377 0 : memset(tlb, 0, bytes);
378 :
379 : /*
380 : * Allocate and initialize the free list array. This array is used
381 : * to find contiguous free memory regions of size up to IO_TLB_SEGSIZE
382 : * between io_tlb_start and io_tlb_end.
383 : */
384 0 : io_tlb_list = (unsigned int *)__get_free_pages(GFP_KERNEL,
385 0 : get_order(io_tlb_nslabs * sizeof(int)));
386 0 : if (!io_tlb_list)
387 0 : goto cleanup3;
388 :
389 0 : io_tlb_orig_addr = (phys_addr_t *)
390 0 : __get_free_pages(GFP_KERNEL,
391 0 : get_order(io_tlb_nslabs *
392 : sizeof(phys_addr_t)));
393 0 : if (!io_tlb_orig_addr)
394 0 : goto cleanup4;
395 :
396 0 : io_tlb_orig_size = (size_t *)
397 0 : __get_free_pages(GFP_KERNEL,
398 0 : get_order(io_tlb_nslabs *
399 : sizeof(size_t)));
400 0 : if (!io_tlb_orig_size)
401 0 : goto cleanup5;
402 :
403 :
404 0 : for (i = 0; i < io_tlb_nslabs; i++) {
405 0 : io_tlb_list[i] = IO_TLB_SEGSIZE - io_tlb_offset(i);
406 0 : io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
407 0 : io_tlb_orig_size[i] = 0;
408 : }
409 0 : io_tlb_index = 0;
410 0 : no_iotlb_memory = false;
411 :
412 0 : swiotlb_print_info();
413 :
414 0 : late_alloc = 1;
415 :
416 0 : swiotlb_set_max_segment(io_tlb_nslabs << IO_TLB_SHIFT);
417 :
418 : return 0;
419 :
420 0 : cleanup5:
421 0 : free_pages((unsigned long)io_tlb_orig_addr, get_order(io_tlb_nslabs *
422 : sizeof(phys_addr_t)));
423 :
424 0 : cleanup4:
425 0 : free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
426 : sizeof(int)));
427 0 : io_tlb_list = NULL;
428 0 : cleanup3:
429 0 : swiotlb_cleanup();
430 0 : return -ENOMEM;
431 : }
432 :
433 0 : void __init swiotlb_exit(void)
434 : {
435 0 : if (!io_tlb_orig_addr)
436 : return;
437 :
438 0 : if (late_alloc) {
439 0 : free_pages((unsigned long)io_tlb_orig_size,
440 0 : get_order(io_tlb_nslabs * sizeof(size_t)));
441 0 : free_pages((unsigned long)io_tlb_orig_addr,
442 0 : get_order(io_tlb_nslabs * sizeof(phys_addr_t)));
443 0 : free_pages((unsigned long)io_tlb_list, get_order(io_tlb_nslabs *
444 : sizeof(int)));
445 0 : free_pages((unsigned long)phys_to_virt(io_tlb_start),
446 0 : get_order(io_tlb_nslabs << IO_TLB_SHIFT));
447 : } else {
448 0 : memblock_free_late(__pa(io_tlb_orig_addr),
449 0 : PAGE_ALIGN(io_tlb_nslabs * sizeof(phys_addr_t)));
450 0 : memblock_free_late(__pa(io_tlb_orig_size),
451 0 : PAGE_ALIGN(io_tlb_nslabs * sizeof(size_t)));
452 0 : memblock_free_late(__pa(io_tlb_list),
453 0 : PAGE_ALIGN(io_tlb_nslabs * sizeof(int)));
454 0 : memblock_free_late(io_tlb_start,
455 0 : PAGE_ALIGN(io_tlb_nslabs << IO_TLB_SHIFT));
456 : }
457 0 : swiotlb_cleanup();
458 : }
459 :
460 : /*
461 : * Bounce: copy the swiotlb buffer from or back to the original dma location
462 : */
463 0 : static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
464 : size_t size, enum dma_data_direction dir)
465 : {
466 0 : unsigned long pfn = PFN_DOWN(orig_addr);
467 0 : unsigned char *vaddr = phys_to_virt(tlb_addr);
468 :
469 0 : if (PageHighMem(pfn_to_page(pfn))) {
470 : /* The buffer does not have a mapping. Map it in and copy */
471 : unsigned int offset = orig_addr & ~PAGE_MASK;
472 : char *buffer;
473 : unsigned int sz = 0;
474 : unsigned long flags;
475 :
476 : while (size) {
477 : sz = min_t(size_t, PAGE_SIZE - offset, size);
478 :
479 : local_irq_save(flags);
480 : buffer = kmap_atomic(pfn_to_page(pfn));
481 : if (dir == DMA_TO_DEVICE)
482 : memcpy(vaddr, buffer + offset, sz);
483 : else
484 : memcpy(buffer + offset, vaddr, sz);
485 : kunmap_atomic(buffer);
486 : local_irq_restore(flags);
487 :
488 : size -= sz;
489 : pfn++;
490 : vaddr += sz;
491 : offset = 0;
492 : }
493 0 : } else if (dir == DMA_TO_DEVICE) {
494 0 : memcpy(vaddr, phys_to_virt(orig_addr), size);
495 : } else {
496 0 : memcpy(phys_to_virt(orig_addr), vaddr, size);
497 : }
498 : }
499 :
500 : #define slot_addr(start, idx) ((start) + ((idx) << IO_TLB_SHIFT))
501 :
502 : /*
503 : * Return the offset into a iotlb slot required to keep the device happy.
504 : */
505 0 : static unsigned int swiotlb_align_offset(struct device *dev, u64 addr)
506 : {
507 0 : return addr & dma_get_min_align_mask(dev) & (IO_TLB_SIZE - 1);
508 : }
509 :
510 : /*
511 : * Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
512 : */
513 0 : static inline unsigned long get_max_slots(unsigned long boundary_mask)
514 : {
515 0 : if (boundary_mask == ~0UL)
516 : return 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
517 0 : return nr_slots(boundary_mask + 1);
518 : }
519 :
520 0 : static unsigned int wrap_index(unsigned int index)
521 : {
522 0 : if (index >= io_tlb_nslabs)
523 0 : return 0;
524 : return index;
525 : }
526 :
527 : /*
528 : * Find a suitable number of IO TLB entries size that will fit this request and
529 : * allocate a buffer from that IO TLB pool.
530 : */
531 0 : static int find_slots(struct device *dev, phys_addr_t orig_addr,
532 : size_t alloc_size)
533 : {
534 0 : unsigned long boundary_mask = dma_get_seg_boundary(dev);
535 0 : dma_addr_t tbl_dma_addr =
536 0 : phys_to_dma_unencrypted(dev, io_tlb_start) & boundary_mask;
537 0 : unsigned long max_slots = get_max_slots(boundary_mask);
538 0 : unsigned int iotlb_align_mask =
539 0 : dma_get_min_align_mask(dev) & ~(IO_TLB_SIZE - 1);
540 0 : unsigned int nslots = nr_slots(alloc_size), stride;
541 0 : unsigned int index, wrap, count = 0, i;
542 0 : unsigned long flags;
543 :
544 0 : BUG_ON(!nslots);
545 :
546 : /*
547 : * For mappings with an alignment requirement don't bother looping to
548 : * unaligned slots once we found an aligned one. For allocations of
549 : * PAGE_SIZE or larger only look for page aligned allocations.
550 : */
551 0 : stride = (iotlb_align_mask >> IO_TLB_SHIFT) + 1;
552 0 : if (alloc_size >= PAGE_SIZE)
553 0 : stride = max(stride, stride << (PAGE_SHIFT - IO_TLB_SHIFT));
554 :
555 0 : spin_lock_irqsave(&io_tlb_lock, flags);
556 0 : if (unlikely(nslots > io_tlb_nslabs - io_tlb_used))
557 0 : goto not_found;
558 :
559 0 : index = wrap = wrap_index(ALIGN(io_tlb_index, stride));
560 0 : do {
561 0 : if ((slot_addr(tbl_dma_addr, index) & iotlb_align_mask) !=
562 : (orig_addr & iotlb_align_mask)) {
563 0 : index = wrap_index(index + 1);
564 0 : continue;
565 : }
566 :
567 : /*
568 : * If we find a slot that indicates we have 'nslots' number of
569 : * contiguous buffers, we allocate the buffers from that slot
570 : * and mark the entries as '0' indicating unavailable.
571 : */
572 0 : if (!iommu_is_span_boundary(index, nslots,
573 : nr_slots(tbl_dma_addr),
574 : max_slots)) {
575 0 : if (io_tlb_list[index] >= nslots)
576 0 : goto found;
577 : }
578 0 : index = wrap_index(index + stride);
579 0 : } while (index != wrap);
580 :
581 0 : not_found:
582 0 : spin_unlock_irqrestore(&io_tlb_lock, flags);
583 0 : return -1;
584 :
585 0 : found:
586 0 : for (i = index; i < index + nslots; i++)
587 0 : io_tlb_list[i] = 0;
588 0 : for (i = index - 1;
589 0 : io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
590 0 : io_tlb_list[i]; i--)
591 0 : io_tlb_list[i] = ++count;
592 :
593 : /*
594 : * Update the indices to avoid searching in the next round.
595 : */
596 0 : if (index + nslots < io_tlb_nslabs)
597 0 : io_tlb_index = index + nslots;
598 : else
599 0 : io_tlb_index = 0;
600 0 : io_tlb_used += nslots;
601 :
602 0 : spin_unlock_irqrestore(&io_tlb_lock, flags);
603 0 : return index;
604 : }
605 :
606 0 : phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
607 : size_t mapping_size, size_t alloc_size,
608 : enum dma_data_direction dir, unsigned long attrs)
609 : {
610 0 : unsigned int offset = swiotlb_align_offset(dev, orig_addr);
611 0 : unsigned int index, i;
612 0 : phys_addr_t tlb_addr;
613 :
614 0 : if (no_iotlb_memory)
615 0 : panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
616 :
617 0 : if (mem_encrypt_active())
618 : pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
619 :
620 0 : if (mapping_size > alloc_size) {
621 0 : dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
622 : mapping_size, alloc_size);
623 0 : return (phys_addr_t)DMA_MAPPING_ERROR;
624 : }
625 :
626 0 : index = find_slots(dev, orig_addr, alloc_size + offset);
627 0 : if (index == -1) {
628 0 : if (!(attrs & DMA_ATTR_NO_WARN))
629 0 : dev_warn_ratelimited(dev,
630 : "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
631 : alloc_size, io_tlb_nslabs, io_tlb_used);
632 0 : return (phys_addr_t)DMA_MAPPING_ERROR;
633 : }
634 :
635 : /*
636 : * Save away the mapping from the original address to the DMA address.
637 : * This is needed when we sync the memory. Then we sync the buffer if
638 : * needed.
639 : */
640 0 : for (i = 0; i < nr_slots(alloc_size + offset); i++) {
641 0 : io_tlb_orig_addr[index + i] = slot_addr(orig_addr, i);
642 0 : io_tlb_orig_size[index+i] = alloc_size - (i << IO_TLB_SHIFT);
643 : }
644 0 : tlb_addr = slot_addr(io_tlb_start, index) + offset;
645 0 : if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
646 : (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
647 0 : swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE);
648 : return tlb_addr;
649 : }
650 :
651 0 : static void validate_sync_size_and_truncate(struct device *hwdev, size_t orig_size, size_t *size)
652 : {
653 0 : if (*size > orig_size) {
654 : /* Warn and truncate mapping_size */
655 0 : dev_WARN_ONCE(hwdev, 1,
656 : "Attempt for buffer overflow. Original size: %zu. Mapping size: %zu.\n",
657 : orig_size, *size);
658 0 : *size = orig_size;
659 : }
660 0 : }
661 :
662 : /*
663 : * tlb_addr is the physical address of the bounce buffer to unmap.
664 : */
665 0 : void swiotlb_tbl_unmap_single(struct device *hwdev, phys_addr_t tlb_addr,
666 : size_t mapping_size, size_t alloc_size,
667 : enum dma_data_direction dir, unsigned long attrs)
668 : {
669 0 : unsigned long flags;
670 0 : unsigned int offset = swiotlb_align_offset(hwdev, tlb_addr);
671 0 : int i, count, nslots = nr_slots(alloc_size + offset);
672 0 : int index = (tlb_addr - offset - io_tlb_start) >> IO_TLB_SHIFT;
673 0 : phys_addr_t orig_addr = io_tlb_orig_addr[index];
674 :
675 0 : validate_sync_size_and_truncate(hwdev, io_tlb_orig_size[index], &mapping_size);
676 :
677 : /*
678 : * First, sync the memory before unmapping the entry
679 : */
680 0 : if (orig_addr != INVALID_PHYS_ADDR &&
681 0 : !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
682 0 : ((dir == DMA_FROM_DEVICE) || (dir == DMA_BIDIRECTIONAL)))
683 0 : swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_FROM_DEVICE);
684 :
685 : /*
686 : * Return the buffer to the free list by setting the corresponding
687 : * entries to indicate the number of contiguous entries available.
688 : * While returning the entries to the free list, we merge the entries
689 : * with slots below and above the pool being returned.
690 : */
691 0 : spin_lock_irqsave(&io_tlb_lock, flags);
692 0 : if (index + nslots < ALIGN(index + 1, IO_TLB_SEGSIZE))
693 0 : count = io_tlb_list[index + nslots];
694 : else
695 : count = 0;
696 :
697 : /*
698 : * Step 1: return the slots to the free list, merging the slots with
699 : * superceeding slots
700 : */
701 0 : for (i = index + nslots - 1; i >= index; i--) {
702 0 : io_tlb_list[i] = ++count;
703 0 : io_tlb_orig_addr[i] = INVALID_PHYS_ADDR;
704 0 : io_tlb_orig_size[i] = 0;
705 : }
706 :
707 : /*
708 : * Step 2: merge the returned slots with the preceding slots, if
709 : * available (non zero)
710 : */
711 0 : for (i = index - 1;
712 0 : io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 && io_tlb_list[i];
713 0 : i--)
714 0 : io_tlb_list[i] = ++count;
715 0 : io_tlb_used -= nslots;
716 0 : spin_unlock_irqrestore(&io_tlb_lock, flags);
717 0 : }
718 :
719 0 : void swiotlb_tbl_sync_single(struct device *hwdev, phys_addr_t tlb_addr,
720 : size_t size, enum dma_data_direction dir,
721 : enum dma_sync_target target)
722 : {
723 0 : int index = (tlb_addr - io_tlb_start) >> IO_TLB_SHIFT;
724 0 : size_t orig_size = io_tlb_orig_size[index];
725 0 : phys_addr_t orig_addr = io_tlb_orig_addr[index];
726 :
727 0 : if (orig_addr == INVALID_PHYS_ADDR)
728 : return;
729 :
730 0 : validate_sync_size_and_truncate(hwdev, orig_size, &size);
731 :
732 0 : switch (target) {
733 0 : case SYNC_FOR_CPU:
734 0 : if (likely(dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL))
735 0 : swiotlb_bounce(orig_addr, tlb_addr,
736 : size, DMA_FROM_DEVICE);
737 : else
738 0 : BUG_ON(dir != DMA_TO_DEVICE);
739 : break;
740 0 : case SYNC_FOR_DEVICE:
741 0 : if (likely(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
742 0 : swiotlb_bounce(orig_addr, tlb_addr,
743 : size, DMA_TO_DEVICE);
744 : else
745 0 : BUG_ON(dir != DMA_FROM_DEVICE);
746 : break;
747 0 : default:
748 0 : BUG();
749 : }
750 : }
751 :
752 : /*
753 : * Create a swiotlb mapping for the buffer at @paddr, and in case of DMAing
754 : * to the device copy the data into it as well.
755 : */
756 0 : dma_addr_t swiotlb_map(struct device *dev, phys_addr_t paddr, size_t size,
757 : enum dma_data_direction dir, unsigned long attrs)
758 : {
759 0 : phys_addr_t swiotlb_addr;
760 0 : dma_addr_t dma_addr;
761 :
762 0 : trace_swiotlb_bounced(dev, phys_to_dma(dev, paddr), size,
763 : swiotlb_force);
764 :
765 0 : swiotlb_addr = swiotlb_tbl_map_single(dev, paddr, size, size, dir,
766 : attrs);
767 0 : if (swiotlb_addr == (phys_addr_t)DMA_MAPPING_ERROR)
768 : return DMA_MAPPING_ERROR;
769 :
770 : /* Ensure that the address returned is DMA'ble */
771 0 : dma_addr = phys_to_dma_unencrypted(dev, swiotlb_addr);
772 0 : if (unlikely(!dma_capable(dev, dma_addr, size, true))) {
773 0 : swiotlb_tbl_unmap_single(dev, swiotlb_addr, size, size, dir,
774 : attrs | DMA_ATTR_SKIP_CPU_SYNC);
775 0 : dev_WARN_ONCE(dev, 1,
776 : "swiotlb addr %pad+%zu overflow (mask %llx, bus limit %llx).\n",
777 : &dma_addr, size, *dev->dma_mask, dev->bus_dma_limit);
778 0 : return DMA_MAPPING_ERROR;
779 : }
780 :
781 0 : if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
782 0 : arch_sync_dma_for_device(swiotlb_addr, size, dir);
783 : return dma_addr;
784 : }
785 :
786 0 : size_t swiotlb_max_mapping_size(struct device *dev)
787 : {
788 0 : return ((size_t)IO_TLB_SIZE) * IO_TLB_SEGSIZE;
789 : }
790 :
791 0 : bool is_swiotlb_active(void)
792 : {
793 : /*
794 : * When SWIOTLB is initialized, even if io_tlb_start points to physical
795 : * address zero, io_tlb_end surely doesn't.
796 : */
797 0 : return io_tlb_end != 0;
798 : }
799 :
800 : #ifdef CONFIG_DEBUG_FS
801 :
802 1 : static int __init swiotlb_create_debugfs(void)
803 : {
804 1 : struct dentry *root;
805 :
806 1 : root = debugfs_create_dir("swiotlb", NULL);
807 1 : debugfs_create_ulong("io_tlb_nslabs", 0400, root, &io_tlb_nslabs);
808 1 : debugfs_create_ulong("io_tlb_used", 0400, root, &io_tlb_used);
809 1 : return 0;
810 : }
811 :
812 : late_initcall(swiotlb_create_debugfs);
813 :
814 : #endif
|
