2016-05-19 00:15:08 +08:00
|
|
|
/*
|
2017-04-12 00:49:49 +08:00
|
|
|
* Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
|
2016-05-19 00:15:08 +08:00
|
|
|
*
|
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
|
* it under the terms of version 2 of the GNU General Public License as
|
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
|
*
|
|
|
|
|
* This program is distributed in the hope that it will be useful, but
|
|
|
|
|
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
|
|
|
|
* General Public License for more details.
|
|
|
|
|
*/
|
|
|
|
|
#include <linux/pagemap.h>
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
|
#include <linux/device.h>
|
|
|
|
|
#include <linux/pfn_t.h>
|
2016-07-25 06:55:42 +08:00
|
|
|
#include <linux/cdev.h>
|
2016-05-19 00:15:08 +08:00
|
|
|
#include <linux/slab.h>
|
|
|
|
|
#include <linux/dax.h>
|
|
|
|
|
#include <linux/fs.h>
|
|
|
|
|
#include <linux/mm.h>
|
2017-04-08 06:33:36 +08:00
|
|
|
#include "dax-private.h"
|
2016-08-07 07:05:06 +08:00
|
|
|
#include "dax.h"
|
2016-05-19 00:15:08 +08:00
|
|
|
|
|
|
|
|
static struct class *dax_class;
|
|
|
|
|
|
2017-04-30 21:57:01 +08:00
|
|
|
/*
|
|
|
|
|
* Rely on the fact that drvdata is set before the attributes are
|
|
|
|
|
* registered, and that the attributes are unregistered before drvdata
|
|
|
|
|
* is cleared to assume that drvdata is always valid.
|
|
|
|
|
*/
|
2016-12-18 06:50:04 +08:00
|
|
|
static ssize_t id_show(struct device *dev,
|
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
|
{
|
2017-04-30 21:57:01 +08:00
|
|
|
struct dax_region *dax_region = dev_get_drvdata(dev);
|
2016-12-18 06:50:04 +08:00
|
|
|
|
2017-04-30 21:57:01 +08:00
|
|
|
return sprintf(buf, "%d\n", dax_region->id);
|
2016-12-18 06:50:04 +08:00
|
|
|
}
|
|
|
|
|
static DEVICE_ATTR_RO(id);
|
|
|
|
|
|
|
|
|
|
static ssize_t region_size_show(struct device *dev,
|
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
|
{
|
2017-04-30 21:57:01 +08:00
|
|
|
struct dax_region *dax_region = dev_get_drvdata(dev);
|
2016-12-18 06:50:04 +08:00
|
|
|
|
2017-04-30 21:57:01 +08:00
|
|
|
return sprintf(buf, "%llu\n", (unsigned long long)
|
|
|
|
|
resource_size(&dax_region->res));
|
2016-12-18 06:50:04 +08:00
|
|
|
}
|
|
|
|
|
static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
|
|
|
|
|
region_size_show, NULL);
|
|
|
|
|
|
|
|
|
|
static ssize_t align_show(struct device *dev,
|
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
|
{
|
2017-04-30 21:57:01 +08:00
|
|
|
struct dax_region *dax_region = dev_get_drvdata(dev);
|
2016-12-18 06:50:04 +08:00
|
|
|
|
2017-04-30 21:57:01 +08:00
|
|
|
return sprintf(buf, "%u\n", dax_region->align);
|
2016-12-18 06:50:04 +08:00
|
|
|
}
|
|
|
|
|
static DEVICE_ATTR_RO(align);
|
|
|
|
|
|
|
|
|
|
static struct attribute *dax_region_attributes[] = {
|
|
|
|
|
&dev_attr_region_size.attr,
|
|
|
|
|
&dev_attr_align.attr,
|
|
|
|
|
&dev_attr_id.attr,
|
|
|
|
|
NULL,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct attribute_group dax_region_attribute_group = {
|
|
|
|
|
.name = "dax_region",
|
|
|
|
|
.attrs = dax_region_attributes,
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct attribute_group *dax_region_attribute_groups[] = {
|
|
|
|
|
&dax_region_attribute_group,
|
|
|
|
|
NULL,
|
|
|
|
|
};
|
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static void dax_region_free(struct kref *kref)
|
|
|
|
|
{
|
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
|
|
|
|
|
dax_region = container_of(kref, struct dax_region, kref);
|
|
|
|
|
kfree(dax_region);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void dax_region_put(struct dax_region *dax_region)
|
|
|
|
|
{
|
|
|
|
|
kref_put(&dax_region->kref, dax_region_free);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(dax_region_put);
|
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
static void dax_region_unregister(void *region)
|
|
|
|
|
{
|
|
|
|
|
struct dax_region *dax_region = region;
|
|
|
|
|
|
|
|
|
|
sysfs_remove_groups(&dax_region->dev->kobj,
|
|
|
|
|
dax_region_attribute_groups);
|
|
|
|
|
dax_region_put(dax_region);
|
|
|
|
|
}
|
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
struct dax_region *alloc_dax_region(struct device *parent, int region_id,
|
|
|
|
|
struct resource *res, unsigned int align, void *addr,
|
|
|
|
|
unsigned long pfn_flags)
|
|
|
|
|
{
|
|
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
/*
|
|
|
|
|
* The DAX core assumes that it can store its private data in
|
|
|
|
|
* parent->driver_data. This WARN is a reminder / safeguard for
|
|
|
|
|
* developers of device-dax drivers.
|
|
|
|
|
*/
|
|
|
|
|
if (dev_get_drvdata(parent)) {
|
|
|
|
|
dev_WARN(parent, "dax core failed to setup private data\n");
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
if (!IS_ALIGNED(res->start, align)
|
|
|
|
|
|| !IS_ALIGNED(resource_size(res), align))
|
|
|
|
|
return NULL;
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
|
2016-05-19 00:15:08 +08:00
|
|
|
if (!dax_region)
|
|
|
|
|
return NULL;
|
|
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
dev_set_drvdata(parent, dax_region);
|
2016-05-19 00:15:08 +08:00
|
|
|
memcpy(&dax_region->res, res, sizeof(*res));
|
|
|
|
|
dax_region->pfn_flags = pfn_flags;
|
|
|
|
|
kref_init(&dax_region->kref);
|
|
|
|
|
dax_region->id = region_id;
|
|
|
|
|
ida_init(&dax_region->ida);
|
|
|
|
|
dax_region->align = align;
|
|
|
|
|
dax_region->dev = parent;
|
|
|
|
|
dax_region->base = addr;
|
2016-12-18 06:50:04 +08:00
|
|
|
if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
|
|
|
|
|
kfree(dax_region);
|
2018-01-23 19:36:32 +08:00
|
|
|
return NULL;
|
2016-12-18 06:50:04 +08:00
|
|
|
}
|
2016-05-19 00:15:08 +08:00
|
|
|
|
2016-12-18 06:50:04 +08:00
|
|
|
kref_get(&dax_region->kref);
|
|
|
|
|
if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
|
|
|
|
|
return NULL;
|
2016-05-19 00:15:08 +08:00
|
|
|
return dax_region;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(alloc_dax_region);
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static struct dev_dax *to_dev_dax(struct device *dev)
|
2016-08-11 15:41:51 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
return container_of(dev, struct dev_dax, dev);
|
2016-08-11 15:41:51 +08:00
|
|
|
}
|
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static ssize_t size_show(struct device *dev,
|
|
|
|
|
struct device_attribute *attr, char *buf)
|
|
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = to_dev_dax(dev);
|
2016-05-19 00:15:08 +08:00
|
|
|
unsigned long long size = 0;
|
|
|
|
|
int i;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
for (i = 0; i < dev_dax->num_resources; i++)
|
|
|
|
|
size += resource_size(&dev_dax->res[i]);
|
2016-05-19 00:15:08 +08:00
|
|
|
|
|
|
|
|
return sprintf(buf, "%llu\n", size);
|
|
|
|
|
}
|
|
|
|
|
static DEVICE_ATTR_RO(size);
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static struct attribute *dev_dax_attributes[] = {
|
2016-05-19 00:15:08 +08:00
|
|
|
&dev_attr_size.attr,
|
|
|
|
|
NULL,
|
|
|
|
|
};
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static const struct attribute_group dev_dax_attribute_group = {
|
|
|
|
|
.attrs = dev_dax_attributes,
|
2016-05-19 00:15:08 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static const struct attribute_group *dax_attribute_groups[] = {
|
2017-01-31 13:43:10 +08:00
|
|
|
&dev_dax_attribute_group,
|
2016-05-19 00:15:08 +08:00
|
|
|
NULL,
|
|
|
|
|
};
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
|
2016-05-15 03:20:44 +08:00
|
|
|
const char *func)
|
|
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dax_region *dax_region = dev_dax->region;
|
|
|
|
|
struct device *dev = &dev_dax->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
unsigned long mask;
|
|
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
if (!dax_alive(dev_dax->dax_dev))
|
2016-05-15 03:20:44 +08:00
|
|
|
return -ENXIO;
|
|
|
|
|
|
2016-11-17 01:00:38 +08:00
|
|
|
/* prevent private mappings from being established */
|
2016-12-07 09:03:35 +08:00
|
|
|
if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
|
2016-05-15 03:20:44 +08:00
|
|
|
dev_info(dev, "%s: %s: fail, attempted private mapping\n",
|
|
|
|
|
current->comm, func);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
mask = dax_region->align - 1;
|
|
|
|
|
if (vma->vm_start & mask || vma->vm_end & mask) {
|
|
|
|
|
dev_info(dev, "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
|
|
|
|
|
current->comm, func, vma->vm_start, vma->vm_end,
|
|
|
|
|
mask);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
|
|
|
|
|
&& (vma->vm_flags & VM_DONTCOPY) == 0) {
|
|
|
|
|
dev_info(dev, "%s: %s: fail, dax range requires MADV_DONTFORK\n",
|
|
|
|
|
current->comm, func);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (!vma_is_dax(vma)) {
|
|
|
|
|
dev_info(dev, "%s: %s: fail, vma is not DAX capable\n",
|
|
|
|
|
current->comm, func);
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-04-08 06:33:36 +08:00
|
|
|
/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
|
2017-05-05 14:38:43 +08:00
|
|
|
__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
|
2016-05-15 03:20:44 +08:00
|
|
|
unsigned long size)
|
|
|
|
|
{
|
|
|
|
|
struct resource *res;
|
2017-10-19 02:21:55 +08:00
|
|
|
/* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */
|
|
|
|
|
phys_addr_t uninitialized_var(phys);
|
2016-05-15 03:20:44 +08:00
|
|
|
int i;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
for (i = 0; i < dev_dax->num_resources; i++) {
|
|
|
|
|
res = &dev_dax->res[i];
|
2016-05-15 03:20:44 +08:00
|
|
|
phys = pgoff * PAGE_SIZE + res->start;
|
|
|
|
|
if (phys >= res->start && phys <= res->end)
|
|
|
|
|
break;
|
|
|
|
|
pgoff -= PHYS_PFN(resource_size(res));
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
if (i < dev_dax->num_resources) {
|
|
|
|
|
res = &dev_dax->res[i];
|
2016-05-15 03:20:44 +08:00
|
|
|
if (phys + size - 1 <= res->end)
|
|
|
|
|
return phys;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct device *dev = &dev_dax->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
int rc = VM_FAULT_SIGBUS;
|
|
|
|
|
phys_addr_t phys;
|
|
|
|
|
pfn_t pfn;
|
2017-03-11 04:24:22 +08:00
|
|
|
unsigned int fault_size = PAGE_SIZE;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
if (check_vma(dev_dax, vmf->vma, __func__))
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dax_region = dev_dax->region;
|
2016-05-15 03:20:44 +08:00
|
|
|
if (dax_region->align > PAGE_SIZE) {
|
2017-04-11 23:59:36 +08:00
|
|
|
dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
|
|
|
|
|
__func__, dax_region->align, fault_size);
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-11 04:24:22 +08:00
|
|
|
if (fault_size != dax_region->align)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-05-05 14:38:43 +08:00
|
|
|
phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
|
2016-05-15 03:20:44 +08:00
|
|
|
if (phys == -1) {
|
2017-03-10 07:56:01 +08:00
|
|
|
dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
|
2016-05-15 03:20:44 +08:00
|
|
|
vmf->pgoff);
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
|
2017-02-25 06:56:41 +08:00
|
|
|
rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
|
|
|
|
if (rc == -ENOMEM)
|
|
|
|
|
return VM_FAULT_OOM;
|
|
|
|
|
if (rc < 0 && rc != -EBUSY)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
|
|
return VM_FAULT_NOPAGE;
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2017-02-23 07:40:03 +08:00
|
|
|
unsigned long pmd_addr = vmf->address & PMD_MASK;
|
2017-01-31 13:43:10 +08:00
|
|
|
struct device *dev = &dev_dax->dev;
|
2016-05-15 03:20:44 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
phys_addr_t phys;
|
|
|
|
|
pgoff_t pgoff;
|
|
|
|
|
pfn_t pfn;
|
2017-03-11 04:24:22 +08:00
|
|
|
unsigned int fault_size = PMD_SIZE;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
if (check_vma(dev_dax, vmf->vma, __func__))
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dax_region = dev_dax->region;
|
2016-05-15 03:20:44 +08:00
|
|
|
if (dax_region->align > PMD_SIZE) {
|
2017-04-11 23:59:36 +08:00
|
|
|
dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
|
|
|
|
|
__func__, dax_region->align, fault_size);
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* dax pmd mappings require pfn_t_devmap() */
|
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
|
2017-04-11 23:59:36 +08:00
|
|
|
dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
|
2016-05-15 03:20:44 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-11 04:24:22 +08:00
|
|
|
if (fault_size < dax_region->align)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
else if (fault_size > dax_region->align)
|
|
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
|
|
|
|
|
|
/* if we are outside of the VMA */
|
|
|
|
|
if (pmd_addr < vmf->vma->vm_start ||
|
|
|
|
|
(pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
pgoff = linear_page_index(vmf->vma, pmd_addr);
|
2017-05-05 14:38:43 +08:00
|
|
|
phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
|
2016-05-15 03:20:44 +08:00
|
|
|
if (phys == -1) {
|
2017-03-10 07:56:01 +08:00
|
|
|
dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
|
2016-05-15 03:20:44 +08:00
|
|
|
pgoff);
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
|
2017-02-23 07:40:06 +08:00
|
|
|
return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
|
2017-02-23 07:40:03 +08:00
|
|
|
vmf->flags & FAULT_FLAG_WRITE);
|
2016-05-15 03:20:44 +08:00
|
|
|
}
|
|
|
|
|
|
2017-02-25 06:57:05 +08:00
|
|
|
#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
|
2017-01-31 13:43:10 +08:00
|
|
|
static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
|
2017-02-25 06:57:05 +08:00
|
|
|
{
|
|
|
|
|
unsigned long pud_addr = vmf->address & PUD_MASK;
|
2017-01-31 13:43:10 +08:00
|
|
|
struct device *dev = &dev_dax->dev;
|
2017-02-25 06:57:05 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
phys_addr_t phys;
|
|
|
|
|
pgoff_t pgoff;
|
|
|
|
|
pfn_t pfn;
|
2017-03-11 04:24:27 +08:00
|
|
|
unsigned int fault_size = PUD_SIZE;
|
|
|
|
|
|
2017-02-25 06:57:05 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
if (check_vma(dev_dax, vmf->vma, __func__))
|
2017-02-25 06:57:05 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dax_region = dev_dax->region;
|
2017-02-25 06:57:05 +08:00
|
|
|
if (dax_region->align > PUD_SIZE) {
|
2017-04-11 23:59:36 +08:00
|
|
|
dev_dbg(dev, "%s: alignment (%#x) > fault size (%#x)\n",
|
|
|
|
|
__func__, dax_region->align, fault_size);
|
2017-02-25 06:57:05 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* dax pud mappings require pfn_t_devmap() */
|
|
|
|
|
if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
|
2017-04-11 23:59:36 +08:00
|
|
|
dev_dbg(dev, "%s: region lacks devmap flags\n", __func__);
|
2017-02-25 06:57:05 +08:00
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
2017-03-11 04:24:27 +08:00
|
|
|
if (fault_size < dax_region->align)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
else if (fault_size > dax_region->align)
|
|
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
|
|
|
|
|
|
/* if we are outside of the VMA */
|
|
|
|
|
if (pud_addr < vmf->vma->vm_start ||
|
|
|
|
|
(pud_addr + PUD_SIZE) > vmf->vma->vm_end)
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
2017-02-25 06:57:05 +08:00
|
|
|
pgoff = linear_page_index(vmf->vma, pud_addr);
|
2017-05-05 14:38:43 +08:00
|
|
|
phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
|
2017-02-25 06:57:05 +08:00
|
|
|
if (phys == -1) {
|
2017-03-10 07:56:01 +08:00
|
|
|
dev_dbg(dev, "%s: pgoff_to_phys(%#lx) failed\n", __func__,
|
2017-02-25 06:57:05 +08:00
|
|
|
pgoff);
|
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
|
|
|
|
|
|
|
|
|
|
return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
|
|
|
|
|
vmf->flags & FAULT_FLAG_WRITE);
|
|
|
|
|
}
|
|
|
|
|
#else
|
2017-01-31 13:43:10 +08:00
|
|
|
static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
|
2017-02-25 06:57:05 +08:00
|
|
|
{
|
|
|
|
|
return VM_FAULT_FALLBACK;
|
|
|
|
|
}
|
|
|
|
|
#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static int dev_dax_huge_fault(struct vm_fault *vmf,
|
2017-02-25 06:57:08 +08:00
|
|
|
enum page_entry_size pe_size)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2017-04-08 07:42:08 +08:00
|
|
|
int rc, id;
|
2017-02-23 07:40:06 +08:00
|
|
|
struct file *filp = vmf->vma->vm_file;
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = filp->private_data;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dbg(&dev_dax->dev, "%s: %s: %s (%#lx - %#lx) size = %d\n", __func__,
|
2017-02-23 07:40:03 +08:00
|
|
|
current->comm, (vmf->flags & FAULT_FLAG_WRITE)
|
2017-02-23 07:40:06 +08:00
|
|
|
? "write" : "read",
|
2017-04-11 23:59:36 +08:00
|
|
|
vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
id = dax_read_lock();
|
2017-02-25 06:57:08 +08:00
|
|
|
switch (pe_size) {
|
|
|
|
|
case PE_SIZE_PTE:
|
2017-01-31 13:43:10 +08:00
|
|
|
rc = __dev_dax_pte_fault(dev_dax, vmf);
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
break;
|
2017-02-25 06:57:08 +08:00
|
|
|
case PE_SIZE_PMD:
|
2017-01-31 13:43:10 +08:00
|
|
|
rc = __dev_dax_pmd_fault(dev_dax, vmf);
|
2017-02-25 06:57:05 +08:00
|
|
|
break;
|
2017-02-25 06:57:08 +08:00
|
|
|
case PE_SIZE_PUD:
|
2017-01-31 13:43:10 +08:00
|
|
|
rc = __dev_dax_pud_fault(dev_dax, vmf);
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
break;
|
|
|
|
|
default:
|
2017-04-12 00:12:25 +08:00
|
|
|
rc = VM_FAULT_SIGBUS;
|
mm,fs,dax: change ->pmd_fault to ->huge_fault
Patch series "1G transparent hugepage support for device dax", v2.
The following series implements support for 1G trasparent hugepage on
x86 for device dax. The bulk of the code was written by Mathew Wilcox a
while back supporting transparent 1G hugepage for fs DAX. I have
forward ported the relevant bits to 4.10-rc. The current submission has
only the necessary code to support device DAX.
Comments from Dan Williams: So the motivation and intended user of this
functionality mirrors the motivation and users of 1GB page support in
hugetlbfs. Given expected capacities of persistent memory devices an
in-memory database may want to reduce tlb pressure beyond what they can
already achieve with 2MB mappings of a device-dax file. We have
customer feedback to that effect as Willy mentioned in his previous
version of these patches [1].
[1]: https://lkml.org/lkml/2016/1/31/52
Comments from Nilesh @ Oracle:
There are applications which have a process model; and if you assume
10,000 processes attempting to mmap all the 6TB memory available on a
server; we are looking at the following:
processes : 10,000
memory : 6TB
pte @ 4k page size: 8 bytes / 4K of memory * #processes = 6TB / 4k * 8 * 10000 = 1.5GB * 80000 = 120,000GB
pmd @ 2M page size: 120,000 / 512 = ~240GB
pud @ 1G page size: 240GB / 512 = ~480MB
As you can see with 2M pages, this system will use up an exorbitant
amount of DRAM to hold the page tables; but the 1G pages finally brings
it down to a reasonable level. Memory sizes will keep increasing; so
this number will keep increasing.
An argument can be made to convert the applications from process model
to thread model, but in the real world that may not be always practical.
Hopefully this helps explain the use case where this is valuable.
This patch (of 3):
In preparation for adding the ability to handle PUD pages, convert
vm_operations_struct.pmd_fault to vm_operations_struct.huge_fault. The
vm_fault structure is extended to include a union of the different page
table pointers that may be needed, and three flag bits are reserved to
indicate which type of pointer is in the union.
[ross.zwisler@linux.intel.com: remove unused function ext4_dax_huge_fault()]
Link: http://lkml.kernel.org/r/1485813172-7284-1-git-send-email-ross.zwisler@linux.intel.com
[dave.jiang@intel.com: clear PMD or PUD size flags when in fall through path]
Link: http://lkml.kernel.org/r/148589842696.5820.16078080610311444794.stgit@djiang5-desk3.ch.intel.com
Link: http://lkml.kernel.org/r/148545058784.17912.6353162518188733642.stgit@djiang5-desk3.ch.intel.com
Signed-off-by: Matthew Wilcox <mawilcox@microsoft.com>
Signed-off-by: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Ross Zwisler <ross.zwisler@linux.intel.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Vlastimil Babka <vbabka@suse.cz>
Cc: Jan Kara <jack@suse.com>
Cc: Dan Williams <dan.j.williams@intel.com>
Cc: Kirill A. Shutemov <kirill.shutemov@linux.intel.com>
Cc: Nilesh Choudhury <nilesh.choudhury@oracle.com>
Cc: Ingo Molnar <mingo@elte.hu>
Cc: "H. Peter Anvin" <hpa@zytor.com>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: Dave Jiang <dave.jiang@intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2017-02-25 06:56:59 +08:00
|
|
|
}
|
2017-04-12 00:49:49 +08:00
|
|
|
dax_read_unlock(id);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
|
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static int dev_dax_fault(struct vm_fault *vmf)
|
2017-02-25 06:57:08 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
|
2017-02-25 06:57:08 +08:00
|
|
|
}
|
|
|
|
|
|
2017-11-30 08:10:32 +08:00
|
|
|
static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
|
|
|
|
|
{
|
|
|
|
|
struct file *filp = vma->vm_file;
|
|
|
|
|
struct dev_dax *dev_dax = filp->private_data;
|
|
|
|
|
struct dax_region *dax_region = dev_dax->region;
|
|
|
|
|
|
|
|
|
|
if (!IS_ALIGNED(addr, dax_region->align))
|
|
|
|
|
return -EINVAL;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static const struct vm_operations_struct dax_vm_ops = {
|
|
|
|
|
.fault = dev_dax_fault,
|
|
|
|
|
.huge_fault = dev_dax_huge_fault,
|
2017-11-30 08:10:32 +08:00
|
|
|
.split = dev_dax_split,
|
2016-05-15 03:20:44 +08:00
|
|
|
};
|
|
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
|
2016-05-15 03:20:44 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = filp->private_data;
|
2017-04-12 00:49:49 +08:00
|
|
|
int rc, id;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dbg(&dev_dax->dev, "%s\n", __func__);
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
/*
|
|
|
|
|
* We lock to check dax_dev liveness and will re-check at
|
|
|
|
|
* fault time.
|
|
|
|
|
*/
|
|
|
|
|
id = dax_read_lock();
|
2017-01-31 13:43:10 +08:00
|
|
|
rc = check_vma(dev_dax, vma, __func__);
|
2017-04-12 00:49:49 +08:00
|
|
|
dax_read_unlock(id);
|
2016-05-15 03:20:44 +08:00
|
|
|
if (rc)
|
|
|
|
|
return rc;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
vma->vm_ops = &dax_vm_ops;
|
2016-05-15 03:20:44 +08:00
|
|
|
vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
|
|
|
|
|
return 0;
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* return an unmapped area aligned to the dax region specified alignment */
|
2016-08-11 15:38:03 +08:00
|
|
|
static unsigned long dax_get_unmapped_area(struct file *filp,
|
2016-08-07 23:23:56 +08:00
|
|
|
unsigned long addr, unsigned long len, unsigned long pgoff,
|
|
|
|
|
unsigned long flags)
|
|
|
|
|
{
|
|
|
|
|
unsigned long off, off_end, off_align, len_align, addr_align, align;
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
|
2016-08-07 23:23:56 +08:00
|
|
|
struct dax_region *dax_region;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
if (!dev_dax || addr)
|
2016-08-07 23:23:56 +08:00
|
|
|
goto out;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dax_region = dev_dax->region;
|
2016-08-07 23:23:56 +08:00
|
|
|
align = dax_region->align;
|
|
|
|
|
off = pgoff << PAGE_SHIFT;
|
|
|
|
|
off_end = off + len;
|
|
|
|
|
off_align = round_up(off, align);
|
|
|
|
|
|
|
|
|
|
if ((off_end <= off_align) || ((off_end - off_align) < align))
|
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
|
|
len_align = len + align;
|
|
|
|
|
if ((off + len_align) < off)
|
|
|
|
|
goto out;
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2016-08-07 23:23:56 +08:00
|
|
|
addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
|
|
|
|
|
pgoff, flags);
|
|
|
|
|
if (!IS_ERR_VALUE(addr_align)) {
|
|
|
|
|
addr_align += (off - addr_align) & (align - 1);
|
|
|
|
|
return addr_align;
|
|
|
|
|
}
|
|
|
|
|
out:
|
|
|
|
|
return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
|
|
|
|
|
}
|
|
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_open(struct inode *inode, struct file *filp)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
2017-04-12 00:49:49 +08:00
|
|
|
struct dax_device *dax_dev = inode_dax(inode);
|
|
|
|
|
struct inode *__dax_inode = dax_inode(dax_dev);
|
|
|
|
|
struct dev_dax *dev_dax = dax_get_private(dax_dev);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dbg(&dev_dax->dev, "%s\n", __func__);
|
2017-04-12 00:49:49 +08:00
|
|
|
inode->i_mapping = __dax_inode->i_mapping;
|
|
|
|
|
inode->i_mapping->host = __dax_inode;
|
2016-07-25 12:55:45 +08:00
|
|
|
filp->f_mapping = inode->i_mapping;
|
fs: new infrastructure for writeback error handling and reporting
Most filesystems currently use mapping_set_error and
filemap_check_errors for setting and reporting/clearing writeback errors
at the mapping level. filemap_check_errors is indirectly called from
most of the filemap_fdatawait_* functions and from
filemap_write_and_wait*. These functions are called from all sorts of
contexts to wait on writeback to finish -- e.g. mostly in fsync, but
also in truncate calls, getattr, etc.
The non-fsync callers are problematic. We should be reporting writeback
errors during fsync, but many places spread over the tree clear out
errors before they can be properly reported, or report errors at
nonsensical times.
If I get -EIO on a stat() call, there is no reason for me to assume that
it is because some previous writeback failed. The fact that it also
clears out the error such that a subsequent fsync returns 0 is a bug,
and a nasty one since that's potentially silent data corruption.
This patch adds a small bit of new infrastructure for setting and
reporting errors during address_space writeback. While the above was my
original impetus for adding this, I think it's also the case that
current fsync semantics are just problematic for userland. Most
applications that call fsync do so to ensure that the data they wrote
has hit the backing store.
In the case where there are multiple writers to the file at the same
time, this is really hard to determine. The first one to call fsync will
see any stored error, and the rest get back 0. The processes with open
fds may not be associated with one another in any way. They could even
be in different containers, so ensuring coordination between all fsync
callers is not really an option.
One way to remedy this would be to track what file descriptor was used
to dirty the file, but that's rather cumbersome and would likely be
slow. However, there is a simpler way to improve the semantics here
without incurring too much overhead.
This set adds an errseq_t to struct address_space, and a corresponding
one is added to struct file. Writeback errors are recorded in the
mapping's errseq_t, and the one in struct file is used as the "since"
value.
This changes the semantics of the Linux fsync implementation such that
applications can now use it to determine whether there were any
writeback errors since fsync(fd) was last called (or since the file was
opened in the case of fsync having never been called).
Note that those writeback errors may have occurred when writing data
that was dirtied via an entirely different fd, but that's the case now
with the current mapping_set_error/filemap_check_error infrastructure.
This will at least prevent you from getting a false report of success.
The new behavior is still consistent with the POSIX spec, and is more
reliable for application developers. This patch just adds some basic
infrastructure for doing this, and ensures that the f_wb_err "cursor"
is properly set when a file is opened. Later patches will change the
existing code to use this new infrastructure for reporting errors at
fsync time.
Signed-off-by: Jeff Layton <jlayton@redhat.com>
Reviewed-by: Jan Kara <jack@suse.cz>
2017-07-06 19:02:25 +08:00
|
|
|
filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
|
2017-01-31 13:43:10 +08:00
|
|
|
filp->private_data = dev_dax;
|
2016-08-11 15:41:51 +08:00
|
|
|
inode->i_flags = S_DAX;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
2016-05-15 03:20:44 +08:00
|
|
|
|
2016-08-11 15:38:03 +08:00
|
|
|
static int dax_release(struct inode *inode, struct file *filp)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = filp->private_data;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dbg(&dev_dax->dev, "%s\n", __func__);
|
2016-08-07 23:23:56 +08:00
|
|
|
return 0;
|
2016-05-15 03:20:44 +08:00
|
|
|
}
|
|
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static const struct file_operations dax_fops = {
|
|
|
|
|
.llseek = noop_llseek,
|
|
|
|
|
.owner = THIS_MODULE,
|
2016-08-11 15:38:03 +08:00
|
|
|
.open = dax_open,
|
|
|
|
|
.release = dax_release,
|
|
|
|
|
.get_unmapped_area = dax_get_unmapped_area,
|
|
|
|
|
.mmap = dax_mmap,
|
2016-05-19 00:15:08 +08:00
|
|
|
};
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static void dev_dax_release(struct device *dev)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = to_dev_dax(dev);
|
|
|
|
|
struct dax_region *dax_region = dev_dax->region;
|
2017-04-12 00:49:49 +08:00
|
|
|
struct dax_device *dax_dev = dev_dax->dax_dev;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-07-19 08:49:14 +08:00
|
|
|
if (dev_dax->id >= 0)
|
|
|
|
|
ida_simple_remove(&dax_region->ida, dev_dax->id);
|
2016-08-11 15:41:51 +08:00
|
|
|
dax_region_put(dax_region);
|
2017-04-12 00:49:49 +08:00
|
|
|
put_dax(dax_dev);
|
2017-01-31 13:43:10 +08:00
|
|
|
kfree(dev_dax);
|
2016-08-11 15:41:51 +08:00
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static void kill_dev_dax(struct dev_dax *dev_dax)
|
2016-08-11 15:41:51 +08:00
|
|
|
{
|
2017-04-12 00:49:49 +08:00
|
|
|
struct dax_device *dax_dev = dev_dax->dax_dev;
|
|
|
|
|
struct inode *inode = dax_inode(dax_dev);
|
|
|
|
|
|
|
|
|
|
kill_dax(dax_dev);
|
|
|
|
|
unmap_mapping_range(inode->i_mapping, 0, 0, 1);
|
2016-08-11 15:41:51 +08:00
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
static void unregister_dev_dax(void *dev)
|
2016-08-11 15:41:51 +08:00
|
|
|
{
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax = to_dev_dax(dev);
|
2017-04-12 00:49:49 +08:00
|
|
|
struct dax_device *dax_dev = dev_dax->dax_dev;
|
|
|
|
|
struct inode *inode = dax_inode(dax_dev);
|
|
|
|
|
struct cdev *cdev = inode->i_cdev;
|
2016-08-11 15:41:51 +08:00
|
|
|
|
2016-08-07 23:23:56 +08:00
|
|
|
dev_dbg(dev, "%s\n", __func__);
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
kill_dev_dax(dev_dax);
|
2017-04-12 00:49:49 +08:00
|
|
|
cdev_device_del(cdev, dev);
|
2017-03-18 02:48:10 +08:00
|
|
|
put_device(dev);
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
|
2017-07-19 08:49:14 +08:00
|
|
|
int id, struct resource *res, int count)
|
2016-08-07 23:23:56 +08:00
|
|
|
{
|
|
|
|
|
struct device *parent = dax_region->dev;
|
2017-04-12 00:49:49 +08:00
|
|
|
struct dax_device *dax_dev;
|
2017-01-31 13:43:10 +08:00
|
|
|
struct dev_dax *dev_dax;
|
2017-04-12 00:49:49 +08:00
|
|
|
struct inode *inode;
|
2016-08-07 23:23:56 +08:00
|
|
|
struct device *dev;
|
2016-07-25 06:55:42 +08:00
|
|
|
struct cdev *cdev;
|
2017-07-13 04:42:37 +08:00
|
|
|
int rc, i;
|
|
|
|
|
|
|
|
|
|
if (!count)
|
|
|
|
|
return ERR_PTR(-EINVAL);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dax = kzalloc(sizeof(*dev_dax) + sizeof(*res) * count, GFP_KERNEL);
|
|
|
|
|
if (!dev_dax)
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(-ENOMEM);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-20 07:17:58 +08:00
|
|
|
for (i = 0; i < count; i++) {
|
|
|
|
|
if (!IS_ALIGNED(res[i].start, dax_region->align)
|
|
|
|
|
|| !IS_ALIGNED(resource_size(&res[i]),
|
|
|
|
|
dax_region->align)) {
|
|
|
|
|
rc = -EINVAL;
|
|
|
|
|
break;
|
|
|
|
|
}
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dax->res[i].start = res[i].start;
|
|
|
|
|
dev_dax->res[i].end = res[i].end;
|
2016-07-20 07:17:58 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (i < count)
|
|
|
|
|
goto err_id;
|
|
|
|
|
|
2017-07-19 08:49:14 +08:00
|
|
|
if (id < 0) {
|
|
|
|
|
id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
|
|
|
|
|
dev_dax->id = id;
|
|
|
|
|
if (id < 0) {
|
|
|
|
|
rc = id;
|
|
|
|
|
goto err_id;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
/* region provider owns @id lifetime */
|
|
|
|
|
dev_dax->id = -1;
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
|
|
|
|
|
2017-01-25 10:44:18 +08:00
|
|
|
/*
|
|
|
|
|
* No 'host' or dax_operations since there is no access to this
|
|
|
|
|
* device outside of mmap of the resulting character device.
|
|
|
|
|
*/
|
|
|
|
|
dax_dev = alloc_dax(dev_dax, NULL, NULL);
|
2017-07-13 04:42:37 +08:00
|
|
|
if (!dax_dev) {
|
|
|
|
|
rc = -ENOMEM;
|
2017-04-12 00:49:49 +08:00
|
|
|
goto err_dax;
|
2017-07-13 04:42:37 +08:00
|
|
|
}
|
2016-07-25 12:55:45 +08:00
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
/* from here on we're committed to teardown via dax_dev_release() */
|
2017-01-31 13:43:10 +08:00
|
|
|
dev = &dev_dax->dev;
|
2016-08-11 15:41:51 +08:00
|
|
|
device_initialize(dev);
|
2016-07-25 06:55:42 +08:00
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
inode = dax_inode(dax_dev);
|
|
|
|
|
cdev = inode->i_cdev;
|
2016-07-25 06:55:42 +08:00
|
|
|
cdev_init(cdev, &dax_fops);
|
|
|
|
|
cdev->owner = parent->driver->owner;
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dax->num_resources = count;
|
2017-04-12 00:49:49 +08:00
|
|
|
dev_dax->dax_dev = dax_dev;
|
2017-01-31 13:43:10 +08:00
|
|
|
dev_dax->region = dax_region;
|
2016-07-25 06:55:42 +08:00
|
|
|
kref_get(&dax_region->kref);
|
|
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
dev->devt = inode->i_rdev;
|
2016-08-11 15:41:51 +08:00
|
|
|
dev->class = dax_class;
|
|
|
|
|
dev->parent = parent;
|
|
|
|
|
dev->groups = dax_attribute_groups;
|
2017-01-31 13:43:10 +08:00
|
|
|
dev->release = dev_dax_release;
|
2017-07-19 08:49:14 +08:00
|
|
|
dev_set_name(dev, "dax%d.%d", dax_region->id, id);
|
2017-03-18 02:48:10 +08:00
|
|
|
|
|
|
|
|
rc = cdev_device_add(cdev, dev);
|
2016-08-11 15:41:51 +08:00
|
|
|
if (rc) {
|
2017-01-31 13:43:10 +08:00
|
|
|
kill_dev_dax(dev_dax);
|
2016-08-11 15:41:51 +08:00
|
|
|
put_device(dev);
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(rc);
|
2016-08-11 15:41:51 +08:00
|
|
|
}
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
|
2016-07-20 08:51:40 +08:00
|
|
|
if (rc)
|
|
|
|
|
return ERR_PTR(rc);
|
|
|
|
|
|
2017-01-31 13:43:10 +08:00
|
|
|
return dev_dax;
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2017-04-12 00:49:49 +08:00
|
|
|
err_dax:
|
2017-07-19 08:49:14 +08:00
|
|
|
if (dev_dax->id >= 0)
|
|
|
|
|
ida_simple_remove(&dax_region->ida, dev_dax->id);
|
2016-08-07 23:23:56 +08:00
|
|
|
err_id:
|
2017-01-31 13:43:10 +08:00
|
|
|
kfree(dev_dax);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-07-20 08:51:40 +08:00
|
|
|
return ERR_PTR(rc);
|
2016-08-07 23:23:56 +08:00
|
|
|
}
|
2017-01-31 13:43:10 +08:00
|
|
|
EXPORT_SYMBOL_GPL(devm_create_dev_dax);
|
2016-08-07 23:23:56 +08:00
|
|
|
|
2016-05-19 00:15:08 +08:00
|
|
|
static int __init dax_init(void)
|
|
|
|
|
{
|
|
|
|
|
dax_class = class_create(THIS_MODULE, "dax");
|
2017-04-12 00:49:49 +08:00
|
|
|
return PTR_ERR_OR_ZERO(dax_class);
|
2016-05-19 00:15:08 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void __exit dax_exit(void)
|
|
|
|
|
{
|
|
|
|
|
class_destroy(dax_class);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
MODULE_AUTHOR("Intel Corporation");
|
|
|
|
|
MODULE_LICENSE("GPL v2");
|
|
|
|
|
subsys_initcall(dax_init);
|
|
|
|
|
module_exit(dax_exit);
|