dma_alloc_wc - notro/udrm-kernel GitHub Wiki

dma_alloc_wc()

static inline void *dma_alloc_wc(struct device *dev, size_t size,
                                 dma_addr_t *dma_addr, gfp_t gfp)
{
        return dma_alloc_attrs(dev, size, dma_addr, gfp,
                               DMA_ATTR_WRITE_COMBINE);
}

dma_alloc_attrs()

static inline void *dma_alloc_attrs(struct device *dev, size_t size,
                                       dma_addr_t *dma_handle, gfp_t flag,
                                       unsigned long attrs)
{
        struct dma_map_ops *ops = get_dma_ops(dev);
        void *cpu_addr;

        BUG_ON(!ops);

        if (dma_alloc_from_coherent(dev, size, dma_handle, &cpu_addr))
                return cpu_addr;

        if (!arch_dma_alloc_attrs(&dev, &flag))
                return NULL;
        if (!ops->alloc)
                return NULL;

        cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);
        debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr);
        return cpu_addr;
}

int dma_alloc_from_coherent(struct device *dev, ssize_t size,
                                       dma_addr_t *dma_handle, void **ret)
{
        struct dma_coherent_mem *mem;
        int order = get_order(size);
        unsigned long flags;
        int pageno;
        int dma_memory_map;

        if (!dev)
                return 0;
        mem = dev->dma_mem;
        if (!mem)
                return 0;
<snip>
}

#define arch_dma_alloc_attrs(dev, flag) (true)

arm_dma_alloc()

/*
 * Allocate DMA-coherent memory space and return both the kernel remapped
 * virtual and bus address for that space.
 */
void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
                    gfp_t gfp, unsigned long attrs)
{
        pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL);

        return __dma_alloc(dev, size, handle, gfp, prot, false,
                           attrs, __builtin_return_address(0));
}

static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot)
{
        prot = (attrs & DMA_ATTR_WRITE_COMBINE) ?
                        pgprot_writecombine(prot) :
                        pgprot_dmacoherent(prot);
        return prot;
}

__dma_alloc()

static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle,
                         gfp_t gfp, pgprot_t prot, bool is_coherent,
                         unsigned long attrs, const void *caller)
{
        u64 mask = get_coherent_dma_mask(dev);
        struct page *page = NULL;
        void *addr;
        bool allowblock, cma;
        struct arm_dma_buffer *buf;
        struct arm_dma_alloc_args args = {
                .dev = dev,
                .size = PAGE_ALIGN(size),
                .gfp = gfp,
                .prot = prot,
                .caller = caller,
                .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0),
                .coherent_flag = is_coherent ? COHERENT : NORMAL,
        };

#ifdef CONFIG_DMA_API_DEBUG
        u64 limit = (mask + 1) & ~mask;
        if (limit && size >= limit) {
                dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n",
                        size, mask);
                return NULL;
        }
#endif

        if (!mask)
                return NULL;

        buf = kzalloc(sizeof(*buf),
                      gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM));
        if (!buf)
                return NULL;

        if (mask < 0xffffffffULL)
                gfp |= GFP_DMA;

        /*
         * Following is a work-around (a.k.a. hack) to prevent pages
         * with __GFP_COMP being passed to split_page() which cannot
         * handle them.  The real problem is that this flag probably
         * should be 0 on ARM as it is not supported on this
         * platform; see CONFIG_HUGETLBFS.
         */
        gfp &= ~(__GFP_COMP);
        args.gfp = gfp;

        *handle = DMA_ERROR_CODE;
        allowblock = gfpflags_allow_blocking(gfp);
        cma = allowblock ? dev_get_cma_area(dev) : false;

        if (cma)
                buf->allocator = &cma_allocator;
        else if (nommu() || is_coherent)
                buf->allocator = &simple_allocator;
        else if (allowblock)
                buf->allocator = &remap_allocator;
        else
                buf->allocator = &pool_allocator;

        addr = buf->allocator->alloc(&args, &page);

        if (page) {
                unsigned long flags;

                *handle = pfn_to_dma(dev, page_to_pfn(page));
                buf->virt = args.want_vaddr ? addr : page;

                spin_lock_irqsave(&arm_dma_bufs_lock, flags);
                list_add(&buf->list, &arm_dma_bufs);
                spin_unlock_irqrestore(&arm_dma_bufs_lock, flags);
        } else {
                kfree(buf);
        }

        return args.want_vaddr ? addr : page;
}

cma_allocator_alloc()

static struct arm_dma_allocator cma_allocator = {
        .alloc = cma_allocator_alloc,
        .free = cma_allocator_free,
};

static void *cma_allocator_alloc(struct arm_dma_alloc_args *args,
                                 struct page **ret_page)
{
        return __alloc_from_contiguous(args->dev, args->size, args->prot,
                                       ret_page, args->caller,
                                       args->want_vaddr, args->coherent_flag);
}

__alloc_from_contiguous()

static void *__alloc_from_contiguous(struct device *dev, size_t size,
                                     pgprot_t prot, struct page **ret_page,
                                     const void *caller, bool want_vaddr,
                                     int coherent_flag)
{
        unsigned long order = get_order(size);
        size_t count = size >> PAGE_SHIFT;
        struct page *page;
        void *ptr = NULL;

        page = dma_alloc_from_contiguous(dev, count, order);
        if (!page)
                return NULL;

        __dma_clear_buffer(page, size, coherent_flag);

        if (!want_vaddr)
                goto out;

        if (PageHighMem(page)) {
                ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller);
                if (!ptr) {
                        dma_release_from_contiguous(dev, page, count);
                        return NULL;
                }
        } else {
                __dma_remap(page, size, prot);
                ptr = page_address(page);
        }

 out:
        *ret_page = page;
        return ptr;
}

dma_alloc_from_contiguous()

/**
 * dma_alloc_from_contiguous() - allocate pages from contiguous area
 * @dev:   Pointer to device for which the allocation is performed.
 * @count: Requested number of pages.
 * @align: Requested alignment of pages (in PAGE_SIZE order).
 *
 * This function allocates memory buffer for specified device. It uses
 * device specific contiguous memory area if available or the default
 * global one. Requires architecture specific dev_get_cma_area() helper
 * function.
 */
struct page *dma_alloc_from_contiguous(struct device *dev, size_t count,
                                       unsigned int align)
{
        if (align > CONFIG_CMA_ALIGNMENT)
                align = CONFIG_CMA_ALIGNMENT;

        return cma_alloc(dev_get_cma_area(dev), count, align);
}

static inline struct cma *dev_get_cma_area(struct device *dev)
{
        if (dev && dev->cma_area)
                return dev->cma_area;
        return dma_contiguous_default_area;
}

start_kernel()->setup_arch()->arm_memblock_init dma_contiguous_reserve(dma_contiguous_default_area)

struct cma *dma_contiguous_default_area;

void __init arm_memblock_init(const struct machine_desc *mdesc)
{
<snip>
        /* reserve memory for DMA contiguous allocations */
        dma_contiguous_reserve(arm_dma_limit);
<snip>
}

#define arm_dma_limit ((phys_addr_t)~0)

void __init dma_contiguous_reserve(phys_addr_t limit)
{
<snip>
                dma_contiguous_reserve_area(selected_size, selected_base,
                                            selected_limit,
                                            &dma_contiguous_default_area,
                                            fixed);
        }
}

cma_alloc()

/**
 * cma_alloc() - allocate pages from contiguous area
 * @cma:   Contiguous memory region for which the allocation is performed.
 * @count: Requested number of pages.
 * @align: Requested alignment of pages (in PAGE_SIZE order).
 *
 * This function allocates part of contiguous memory on specific
 * contiguous memory area.
 */
struct page *cma_alloc(struct cma *cma, size_t count, unsigned int align)
{
        unsigned long mask, offset;
        unsigned long pfn = -1;
        unsigned long start = 0;
        unsigned long bitmap_maxno, bitmap_no, bitmap_count;
        struct page *page = NULL;
        int ret;

        if (!cma || !cma->count)
                return NULL;

        pr_debug("%s(cma %p, count %zu, align %d)\n", __func__, (void *)cma,
                 count, align);

        if (!count)
                return NULL;

        mask = cma_bitmap_aligned_mask(cma, align);
        offset = cma_bitmap_aligned_offset(cma, align);
        bitmap_maxno = cma_bitmap_maxno(cma);
        bitmap_count = cma_bitmap_pages_to_bits(cma, count);

        if (bitmap_count > bitmap_maxno)
                return NULL;

        for (;;) {
                mutex_lock(&cma->lock);
                bitmap_no = bitmap_find_next_zero_area_off(cma->bitmap,
                                bitmap_maxno, start, bitmap_count, mask,
                                offset);
                if (bitmap_no >= bitmap_maxno) {
                        mutex_unlock(&cma->lock);
                        break;
                }
                bitmap_set(cma->bitmap, bitmap_no, bitmap_count);
                /*
                 * It's safe to drop the lock here. We've marked this region for
                 * our exclusive use. If the migration fails we will take the
                 * lock again and unmark it.
                 */
                mutex_unlock(&cma->lock);

                pfn = cma->base_pfn + (bitmap_no << cma->order_per_bit);
                mutex_lock(&cma_mutex);
                ret = alloc_contig_range(pfn, pfn + count, MIGRATE_CMA);
                mutex_unlock(&cma_mutex);
                if (ret == 0) {
                        page = pfn_to_page(pfn);
                        break;
                }

                cma_clear_bitmap(cma, pfn, count);
                if (ret != -EBUSY)
                        break;

                pr_debug("%s(): memory range at %p is busy, retrying\n",
                         __func__, pfn_to_page(pfn));
                /* try again with a bit different memory target */
                start = bitmap_no + mask + 1;
        }

        trace_cma_alloc(pfn, page, count, align);

        pr_debug("%s(): returned %p\n", __func__, page);
        return page;
}
⚠️ **GitHub.com Fallback** ⚠️