2007-07-19 16:49:33 +08:00
|
|
|
/*
|
|
|
|
* edac_mc kernel module
|
2007-07-19 16:50:10 +08:00
|
|
|
* (C) 2005-2007 Linux Networx (http://lnxi.com)
|
|
|
|
*
|
2007-07-19 16:49:33 +08:00
|
|
|
* This file may be distributed under the terms of the
|
|
|
|
* GNU General Public License.
|
|
|
|
*
|
2007-07-19 16:50:10 +08:00
|
|
|
* Written Doug Thompson <norsk5@xmission.com> www.softwarebitmaker.com
|
2007-07-19 16:49:33 +08:00
|
|
|
*
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/ctype.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 16:04:11 +08:00
|
|
|
#include <linux/slab.h>
|
2010-09-02 23:26:48 +08:00
|
|
|
#include <linux/edac.h>
|
2007-07-19 16:50:27 +08:00
|
|
|
#include <linux/bug.h>
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2007-07-19 16:49:47 +08:00
|
|
|
#include "edac_core.h"
|
2007-07-19 16:49:33 +08:00
|
|
|
#include "edac_module.h"
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
/* MC EDAC Controls, setable by module parameter, and sysfs */
|
2007-07-19 16:49:54 +08:00
|
|
|
static int edac_mc_log_ue = 1;
|
|
|
|
static int edac_mc_log_ce = 1;
|
2007-07-19 16:50:19 +08:00
|
|
|
static int edac_mc_panic_on_ue;
|
2007-07-19 16:49:54 +08:00
|
|
|
static int edac_mc_poll_msec = 1000;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* Getter functions for above */
|
2007-07-19 16:49:54 +08:00
|
|
|
int edac_mc_get_log_ue(void)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:54 +08:00
|
|
|
return edac_mc_log_ue;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:54 +08:00
|
|
|
int edac_mc_get_log_ce(void)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:54 +08:00
|
|
|
return edac_mc_log_ce;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:54 +08:00
|
|
|
int edac_mc_get_panic_on_ue(void)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:54 +08:00
|
|
|
return edac_mc_panic_on_ue;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:52 +08:00
|
|
|
/* this is temporary */
|
|
|
|
int edac_mc_get_poll_msec(void)
|
|
|
|
{
|
2007-07-19 16:49:54 +08:00
|
|
|
return edac_mc_poll_msec;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2008-07-25 16:49:09 +08:00
|
|
|
static int edac_set_poll_msec(const char *val, struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
long l;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!val)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
ret = strict_strtol(val, 0, &l);
|
|
|
|
if (ret == -EINVAL || ((int)l != l))
|
|
|
|
return -EINVAL;
|
|
|
|
*((int *)kp->arg) = l;
|
|
|
|
|
|
|
|
/* notify edac_mc engine to reset the poll period */
|
|
|
|
edac_mc_reset_delay_period(l);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
/* Parameter declarations for above */
|
2007-07-19 16:49:54 +08:00
|
|
|
module_param(edac_mc_panic_on_ue, int, 0644);
|
|
|
|
MODULE_PARM_DESC(edac_mc_panic_on_ue, "Panic on uncorrected error: 0=off 1=on");
|
|
|
|
module_param(edac_mc_log_ue, int, 0644);
|
|
|
|
MODULE_PARM_DESC(edac_mc_log_ue,
|
2007-07-19 16:49:58 +08:00
|
|
|
"Log uncorrectable error to console: 0=off 1=on");
|
2007-07-19 16:49:54 +08:00
|
|
|
module_param(edac_mc_log_ce, int, 0644);
|
|
|
|
MODULE_PARM_DESC(edac_mc_log_ce,
|
2007-07-19 16:49:58 +08:00
|
|
|
"Log correctable error to console: 0=off 1=on");
|
2008-07-25 16:49:09 +08:00
|
|
|
module_param_call(edac_mc_poll_msec, edac_set_poll_msec, param_get_int,
|
|
|
|
&edac_mc_poll_msec, 0644);
|
2007-07-19 16:49:54 +08:00
|
|
|
MODULE_PARM_DESC(edac_mc_poll_msec, "Polling period in milliseconds");
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* various constants for Memory Controllers
|
|
|
|
*/
|
|
|
|
static const char *mem_types[] = {
|
|
|
|
[MEM_EMPTY] = "Empty",
|
|
|
|
[MEM_RESERVED] = "Reserved",
|
|
|
|
[MEM_UNKNOWN] = "Unknown",
|
|
|
|
[MEM_FPM] = "FPM",
|
|
|
|
[MEM_EDO] = "EDO",
|
|
|
|
[MEM_BEDO] = "BEDO",
|
|
|
|
[MEM_SDR] = "Unbuffered-SDR",
|
|
|
|
[MEM_RDR] = "Registered-SDR",
|
|
|
|
[MEM_DDR] = "Unbuffered-DDR",
|
|
|
|
[MEM_RDDR] = "Registered-DDR",
|
2007-07-19 16:49:38 +08:00
|
|
|
[MEM_RMBS] = "RMBS",
|
|
|
|
[MEM_DDR2] = "Unbuffered-DDR2",
|
|
|
|
[MEM_FB_DDR2] = "FullyBuffered-DDR2",
|
2008-02-07 16:14:52 +08:00
|
|
|
[MEM_RDDR2] = "Registered-DDR2",
|
2009-07-01 02:41:22 +08:00
|
|
|
[MEM_XDR] = "XDR",
|
|
|
|
[MEM_DDR3] = "Unbuffered-DDR3",
|
|
|
|
[MEM_RDDR3] = "Registered-DDR3"
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
static const char *dev_types[] = {
|
|
|
|
[DEV_UNKNOWN] = "Unknown",
|
|
|
|
[DEV_X1] = "x1",
|
|
|
|
[DEV_X2] = "x2",
|
|
|
|
[DEV_X4] = "x4",
|
|
|
|
[DEV_X8] = "x8",
|
|
|
|
[DEV_X16] = "x16",
|
|
|
|
[DEV_X32] = "x32",
|
|
|
|
[DEV_X64] = "x64"
|
|
|
|
};
|
|
|
|
|
|
|
|
static const char *edac_caps[] = {
|
|
|
|
[EDAC_UNKNOWN] = "Unknown",
|
|
|
|
[EDAC_NONE] = "None",
|
|
|
|
[EDAC_RESERVED] = "Reserved",
|
|
|
|
[EDAC_PARITY] = "PARITY",
|
|
|
|
[EDAC_EC] = "EC",
|
|
|
|
[EDAC_SECDED] = "SECDED",
|
|
|
|
[EDAC_S2ECD2ED] = "S2ECD2ED",
|
|
|
|
[EDAC_S4ECD4ED] = "S4ECD4ED",
|
|
|
|
[EDAC_S8ECD8ED] = "S8ECD8ED",
|
|
|
|
[EDAC_S16ECD16ED] = "S16ECD16ED"
|
|
|
|
};
|
|
|
|
|
|
|
|
/* EDAC sysfs CSROW data structures and methods
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Set of more default csrow<id> attribute show/store functions */
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_ue_count_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%u\n", csrow->ue_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_ce_count_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%u\n", csrow->ce_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_size_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%u\n", PAGES_TO_MiB(csrow->nr_pages));
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_mem_type_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%s\n", mem_types[csrow->mtype]);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_dev_type_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%s\n", dev_types[csrow->dtype]);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:36 +08:00
|
|
|
static ssize_t csrow_edac_mode_show(struct csrow_info *csrow, char *data,
|
2007-07-19 16:50:13 +08:00
|
|
|
int private)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%s\n", edac_caps[csrow->edac_mode]);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* show/store functions for DIMM Label attributes */
|
|
|
|
static ssize_t channel_dimm_label_show(struct csrow_info *csrow,
|
2007-07-19 16:50:13 +08:00
|
|
|
char *data, int channel)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2008-07-25 16:49:12 +08:00
|
|
|
/* if field has not been initialized, there is nothing to send */
|
|
|
|
if (!csrow->channels[channel].label[0])
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
return snprintf(data, EDAC_MC_LABEL_LEN, "%s\n",
|
2007-07-19 16:49:33 +08:00
|
|
|
csrow->channels[channel].label);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t channel_dimm_label_store(struct csrow_info *csrow,
|
2007-07-19 16:49:58 +08:00
|
|
|
const char *data,
|
|
|
|
size_t count, int channel)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
ssize_t max_size = 0;
|
|
|
|
|
2007-07-19 16:49:58 +08:00
|
|
|
max_size = min((ssize_t) count, (ssize_t) EDAC_MC_LABEL_LEN - 1);
|
2007-07-19 16:49:33 +08:00
|
|
|
strncpy(csrow->channels[channel].label, data, max_size);
|
|
|
|
csrow->channels[channel].label[max_size] = '\0';
|
|
|
|
|
|
|
|
return max_size;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* show function for dynamic chX_ce_count attribute */
|
|
|
|
static ssize_t channel_ce_count_show(struct csrow_info *csrow,
|
2007-07-19 16:50:13 +08:00
|
|
|
char *data, int channel)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
return sprintf(data, "%u\n", csrow->channels[channel].ce_count);
|
|
|
|
}
|
|
|
|
|
|
|
|
/* csrow specific attribute structure */
|
|
|
|
struct csrowdev_attribute {
|
|
|
|
struct attribute attr;
|
2007-07-19 16:49:58 +08:00
|
|
|
ssize_t(*show) (struct csrow_info *, char *, int);
|
|
|
|
ssize_t(*store) (struct csrow_info *, const char *, size_t, int);
|
|
|
|
int private;
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#define to_csrow(k) container_of(k, struct csrow_info, kobj)
|
|
|
|
#define to_csrowdev_attr(a) container_of(a, struct csrowdev_attribute, attr)
|
|
|
|
|
|
|
|
/* Set of show/store higher level functions for default csrow attributes */
|
|
|
|
static ssize_t csrowdev_show(struct kobject *kobj,
|
2007-07-19 16:50:13 +08:00
|
|
|
struct attribute *attr, char *buffer)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
struct csrow_info *csrow = to_csrow(kobj);
|
|
|
|
struct csrowdev_attribute *csrowdev_attr = to_csrowdev_attr(attr);
|
|
|
|
|
|
|
|
if (csrowdev_attr->show)
|
|
|
|
return csrowdev_attr->show(csrow,
|
2007-07-19 16:50:13 +08:00
|
|
|
buffer, csrowdev_attr->private);
|
2007-07-19 16:49:33 +08:00
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t csrowdev_store(struct kobject *kobj, struct attribute *attr,
|
2007-07-19 16:50:13 +08:00
|
|
|
const char *buffer, size_t count)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
struct csrow_info *csrow = to_csrow(kobj);
|
2007-07-19 16:49:58 +08:00
|
|
|
struct csrowdev_attribute *csrowdev_attr = to_csrowdev_attr(attr);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
if (csrowdev_attr->store)
|
|
|
|
return csrowdev_attr->store(csrow,
|
2007-07-19 16:50:13 +08:00
|
|
|
buffer,
|
|
|
|
count, csrowdev_attr->private);
|
2007-07-19 16:49:33 +08:00
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
2010-01-19 09:58:23 +08:00
|
|
|
static const struct sysfs_ops csrowfs_ops = {
|
2007-07-19 16:49:58 +08:00
|
|
|
.show = csrowdev_show,
|
|
|
|
.store = csrowdev_store
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#define CSROWDEV_ATTR(_name,_mode,_show,_store,_private) \
|
|
|
|
static struct csrowdev_attribute attr_##_name = { \
|
|
|
|
.attr = {.name = __stringify(_name), .mode = _mode }, \
|
|
|
|
.show = _show, \
|
|
|
|
.store = _store, \
|
|
|
|
.private = _private, \
|
|
|
|
};
|
|
|
|
|
|
|
|
/* default cwrow<id>/attribute files */
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(size_mb, S_IRUGO, csrow_size_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(dev_type, S_IRUGO, csrow_dev_type_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(mem_type, S_IRUGO, csrow_mem_type_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(edac_mode, S_IRUGO, csrow_edac_mode_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(ue_count, S_IRUGO, csrow_ue_count_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(ce_count, S_IRUGO, csrow_ce_count_show, NULL, 0);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* default attributes of the CSROW<id> object */
|
|
|
|
static struct csrowdev_attribute *default_csrow_attr[] = {
|
|
|
|
&attr_dev_type,
|
|
|
|
&attr_mem_type,
|
|
|
|
&attr_edac_mode,
|
|
|
|
&attr_size_mb,
|
|
|
|
&attr_ue_count,
|
|
|
|
&attr_ce_count,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
|
|
|
/* possible dynamic channel DIMM Label attribute files */
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch0_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 0);
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch1_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 1);
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch2_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 2);
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch3_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 3);
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch4_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 4);
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch5_dimm_label, S_IRUGO | S_IWUSR,
|
2007-07-19 16:50:13 +08:00
|
|
|
channel_dimm_label_show, channel_dimm_label_store, 5);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* Total possible dynamic DIMM Label attribute file table */
|
|
|
|
static struct csrowdev_attribute *dynamic_csrow_dimm_attr[] = {
|
2007-07-19 16:49:58 +08:00
|
|
|
&attr_ch0_dimm_label,
|
|
|
|
&attr_ch1_dimm_label,
|
|
|
|
&attr_ch2_dimm_label,
|
|
|
|
&attr_ch3_dimm_label,
|
|
|
|
&attr_ch4_dimm_label,
|
|
|
|
&attr_ch5_dimm_label
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
/* possible dynamic channel ce_count attribute files */
|
2007-07-19 16:49:58 +08:00
|
|
|
CSROWDEV_ATTR(ch0_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 0);
|
|
|
|
CSROWDEV_ATTR(ch1_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 1);
|
|
|
|
CSROWDEV_ATTR(ch2_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 2);
|
|
|
|
CSROWDEV_ATTR(ch3_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 3);
|
|
|
|
CSROWDEV_ATTR(ch4_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 4);
|
|
|
|
CSROWDEV_ATTR(ch5_ce_count, S_IRUGO | S_IWUSR, channel_ce_count_show, NULL, 5);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* Total possible dynamic ce_count attribute file table */
|
|
|
|
static struct csrowdev_attribute *dynamic_csrow_ce_count_attr[] = {
|
2007-07-19 16:49:58 +08:00
|
|
|
&attr_ch0_ce_count,
|
|
|
|
&attr_ch1_ce_count,
|
|
|
|
&attr_ch2_ce_count,
|
|
|
|
&attr_ch3_ce_count,
|
|
|
|
&attr_ch4_ce_count,
|
|
|
|
&attr_ch5_ce_count
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
#define EDAC_NR_CHANNELS 6
|
|
|
|
|
|
|
|
/* Create dynamic CHANNEL files, indexed by 'chan', under specifed CSROW */
|
|
|
|
static int edac_create_channel_files(struct kobject *kobj, int chan)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
int err = -ENODEV;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
if (chan >= EDAC_NR_CHANNELS)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
/* create the DIMM label attribute file */
|
|
|
|
err = sysfs_create_file(kobj,
|
2007-07-19 16:49:58 +08:00
|
|
|
(struct attribute *)
|
|
|
|
dynamic_csrow_dimm_attr[chan]);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
if (!err) {
|
|
|
|
/* create the CE Count attribute file */
|
|
|
|
err = sysfs_create_file(kobj,
|
2007-07-19 16:49:58 +08:00
|
|
|
(struct attribute *)
|
|
|
|
dynamic_csrow_ce_count_attr[chan]);
|
2007-07-19 16:49:33 +08:00
|
|
|
} else {
|
2007-07-19 16:49:36 +08:00
|
|
|
debugf1("%s() dimm labels and ce_count files created",
|
|
|
|
__func__);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No memory to release for this kobj */
|
|
|
|
static void edac_csrow_instance_release(struct kobject *kobj)
|
|
|
|
{
|
2007-07-19 16:50:27 +08:00
|
|
|
struct mem_ctl_info *mci;
|
2007-07-19 16:49:33 +08:00
|
|
|
struct csrow_info *cs;
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
cs = container_of(kobj, struct csrow_info, kobj);
|
2007-07-19 16:50:27 +08:00
|
|
|
mci = cs->mci;
|
|
|
|
|
|
|
|
kobject_put(&mci->edac_mci_kobj);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* the kobj_type instance for a CSROW */
|
|
|
|
static struct kobj_type ktype_csrow = {
|
|
|
|
.release = edac_csrow_instance_release,
|
|
|
|
.sysfs_ops = &csrowfs_ops,
|
2007-07-19 16:49:58 +08:00
|
|
|
.default_attrs = (struct attribute **)default_csrow_attr,
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
|
|
|
/* Create a CSROW object under specifed edac_mc_device */
|
2007-07-19 16:50:27 +08:00
|
|
|
static int edac_create_csrow_object(struct mem_ctl_info *mci,
|
|
|
|
struct csrow_info *csrow, int index)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2007-07-19 16:50:27 +08:00
|
|
|
struct kobject *kobj_mci = &mci->edac_mci_kobj;
|
|
|
|
struct kobject *kobj;
|
2007-07-19 16:49:33 +08:00
|
|
|
int chan;
|
2007-07-19 16:50:27 +08:00
|
|
|
int err;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* generate ..../edac/mc/mc<id>/csrow<index> */
|
2007-07-19 16:50:27 +08:00
|
|
|
memset(&csrow->kobj, 0, sizeof(csrow->kobj));
|
|
|
|
csrow->mci = mci; /* include container up link */
|
|
|
|
|
|
|
|
/* bump the mci instance's kobject's ref count */
|
|
|
|
kobj = kobject_get(&mci->edac_mci_kobj);
|
|
|
|
if (!kobj) {
|
|
|
|
err = -ENODEV;
|
|
|
|
goto err_out;
|
|
|
|
}
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* Instanstiate the csrow object */
|
2007-12-18 03:54:39 +08:00
|
|
|
err = kobject_init_and_add(&csrow->kobj, &ktype_csrow, kobj_mci,
|
|
|
|
"csrow%d", index);
|
2007-07-19 16:50:27 +08:00
|
|
|
if (err)
|
|
|
|
goto err_release_top_kobj;
|
|
|
|
|
|
|
|
/* At this point, to release a csrow kobj, one must
|
2007-12-21 00:13:05 +08:00
|
|
|
* call the kobject_put and allow that tear down
|
2007-07-19 16:50:27 +08:00
|
|
|
* to work the releasing
|
|
|
|
*/
|
|
|
|
|
|
|
|
/* Create the dyanmic attribute files on this csrow,
|
|
|
|
* namely, the DIMM labels and the channel ce_count
|
|
|
|
*/
|
|
|
|
for (chan = 0; chan < csrow->nr_channels; chan++) {
|
|
|
|
err = edac_create_channel_files(&csrow->kobj, chan);
|
|
|
|
if (err) {
|
|
|
|
/* special case the unregister here */
|
2007-12-21 00:13:05 +08:00
|
|
|
kobject_put(&csrow->kobj);
|
2007-07-19 16:50:27 +08:00
|
|
|
goto err_out;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
}
|
2007-12-18 03:54:39 +08:00
|
|
|
kobject_uevent(&csrow->kobj, KOBJ_ADD);
|
2007-07-19 16:50:27 +08:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* error unwind stack */
|
|
|
|
err_release_top_kobj:
|
|
|
|
kobject_put(&mci->edac_mci_kobj);
|
|
|
|
|
|
|
|
err_out:
|
2007-07-19 16:49:33 +08:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* default sysfs methods and data structures for the main MCI kobject */
|
|
|
|
|
|
|
|
static ssize_t mci_reset_counters_store(struct mem_ctl_info *mci,
|
2007-07-19 16:49:58 +08:00
|
|
|
const char *data, size_t count)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
int row, chan;
|
|
|
|
|
|
|
|
mci->ue_noinfo_count = 0;
|
|
|
|
mci->ce_noinfo_count = 0;
|
|
|
|
mci->ue_count = 0;
|
|
|
|
mci->ce_count = 0;
|
|
|
|
|
|
|
|
for (row = 0; row < mci->nr_csrows; row++) {
|
|
|
|
struct csrow_info *ri = &mci->csrows[row];
|
|
|
|
|
|
|
|
ri->ue_count = 0;
|
|
|
|
ri->ce_count = 0;
|
|
|
|
|
|
|
|
for (chan = 0; chan < ri->nr_channels; chan++)
|
|
|
|
ri->channels[chan].ce_count = 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
mci->start_time = jiffies;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
/* Memory scrubbing interface:
|
|
|
|
*
|
|
|
|
* A MC driver can limit the scrubbing bandwidth based on the CPU type.
|
|
|
|
* Therefore, ->set_sdram_scrub_rate should be made to return the actual
|
|
|
|
* bandwidth that is accepted or 0 when scrubbing is to be disabled.
|
|
|
|
*
|
|
|
|
* Negative value still means that an error has occurred while setting
|
|
|
|
* the scrub rate.
|
|
|
|
*/
|
2007-07-19 16:49:33 +08:00
|
|
|
static ssize_t mci_sdram_scrub_rate_store(struct mem_ctl_info *mci,
|
2010-05-26 00:21:07 +08:00
|
|
|
const char *data, size_t count)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
2010-05-26 00:21:07 +08:00
|
|
|
unsigned long bandwidth = 0;
|
2010-11-25 02:52:09 +08:00
|
|
|
int new_bw = 0;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
if (!mci->set_sdram_scrub_rate)
|
2010-05-26 00:21:07 +08:00
|
|
|
return -EINVAL;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2010-05-26 00:21:07 +08:00
|
|
|
if (strict_strtoul(data, 10, &bandwidth) < 0)
|
|
|
|
return -EINVAL;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
new_bw = mci->set_sdram_scrub_rate(mci, bandwidth);
|
2011-04-21 02:28:45 +08:00
|
|
|
if (new_bw < 0) {
|
|
|
|
edac_printk(KERN_WARNING, EDAC_MC,
|
|
|
|
"Error setting scrub rate to: %lu\n", bandwidth);
|
|
|
|
return -EINVAL;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
2010-11-25 02:52:09 +08:00
|
|
|
|
2011-04-21 02:28:45 +08:00
|
|
|
return count;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
/*
|
|
|
|
* ->get_sdram_scrub_rate() return value semantics same as above.
|
|
|
|
*/
|
2007-07-19 16:49:33 +08:00
|
|
|
static ssize_t mci_sdram_scrub_rate_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2010-11-25 02:52:09 +08:00
|
|
|
int bandwidth = 0;
|
2010-05-26 00:21:07 +08:00
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
if (!mci->get_sdram_scrub_rate)
|
2010-05-26 00:21:07 +08:00
|
|
|
return -EINVAL;
|
|
|
|
|
2010-11-25 02:52:09 +08:00
|
|
|
bandwidth = mci->get_sdram_scrub_rate(mci);
|
|
|
|
if (bandwidth < 0) {
|
2010-05-26 00:21:07 +08:00
|
|
|
edac_printk(KERN_DEBUG, EDAC_MC, "Error reading scrub rate\n");
|
2010-11-25 02:52:09 +08:00
|
|
|
return bandwidth;
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
2010-11-25 02:52:09 +08:00
|
|
|
|
|
|
|
return sprintf(data, "%d\n", bandwidth);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
/* default attribute files for the MCI object */
|
|
|
|
static ssize_t mci_ue_count_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%d\n", mci->ue_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_ce_count_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%d\n", mci->ce_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_ce_noinfo_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%d\n", mci->ce_noinfo_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_ue_noinfo_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%d\n", mci->ue_noinfo_count);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_seconds_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%ld\n", (jiffies - mci->start_time) / HZ);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_ctl_name_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%s\n", mci->ctl_name);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mci_size_mb_show(struct mem_ctl_info *mci, char *data)
|
|
|
|
{
|
|
|
|
int total_pages, csrow_idx;
|
|
|
|
|
|
|
|
for (total_pages = csrow_idx = 0; csrow_idx < mci->nr_csrows;
|
2007-07-19 16:50:13 +08:00
|
|
|
csrow_idx++) {
|
2007-07-19 16:49:33 +08:00
|
|
|
struct csrow_info *csrow = &mci->csrows[csrow_idx];
|
|
|
|
|
|
|
|
if (!csrow->nr_pages)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
total_pages += csrow->nr_pages;
|
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:58 +08:00
|
|
|
return sprintf(data, "%u\n", PAGES_TO_MiB(total_pages));
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
#define to_mci(k) container_of(k, struct mem_ctl_info, edac_mci_kobj)
|
2007-07-19 16:50:10 +08:00
|
|
|
#define to_mcidev_attr(a) container_of(a,struct mcidev_sysfs_attribute,attr)
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* MCI show/store functions for top most object */
|
|
|
|
static ssize_t mcidev_show(struct kobject *kobj, struct attribute *attr,
|
2007-07-19 16:50:13 +08:00
|
|
|
char *buffer)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
struct mem_ctl_info *mem_ctl_info = to_mci(kobj);
|
2007-07-19 16:50:10 +08:00
|
|
|
struct mcidev_sysfs_attribute *mcidev_attr = to_mcidev_attr(attr);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2009-09-25 03:23:42 +08:00
|
|
|
debugf1("%s() mem_ctl_info %p\n", __func__, mem_ctl_info);
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
if (mcidev_attr->show)
|
|
|
|
return mcidev_attr->show(mem_ctl_info, buffer);
|
|
|
|
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t mcidev_store(struct kobject *kobj, struct attribute *attr,
|
2007-07-19 16:50:13 +08:00
|
|
|
const char *buffer, size_t count)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
struct mem_ctl_info *mem_ctl_info = to_mci(kobj);
|
2007-07-19 16:50:10 +08:00
|
|
|
struct mcidev_sysfs_attribute *mcidev_attr = to_mcidev_attr(attr);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2009-09-25 03:23:42 +08:00
|
|
|
debugf1("%s() mem_ctl_info %p\n", __func__, mem_ctl_info);
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
if (mcidev_attr->store)
|
|
|
|
return mcidev_attr->store(mem_ctl_info, buffer, count);
|
|
|
|
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* Intermediate show/store table */
|
2010-01-19 09:58:23 +08:00
|
|
|
static const struct sysfs_ops mci_ops = {
|
2007-07-19 16:49:33 +08:00
|
|
|
.show = mcidev_show,
|
|
|
|
.store = mcidev_store
|
|
|
|
};
|
|
|
|
|
|
|
|
#define MCIDEV_ATTR(_name,_mode,_show,_store) \
|
2007-07-19 16:50:10 +08:00
|
|
|
static struct mcidev_sysfs_attribute mci_attr_##_name = { \
|
2007-07-19 16:49:33 +08:00
|
|
|
.attr = {.name = __stringify(_name), .mode = _mode }, \
|
|
|
|
.show = _show, \
|
|
|
|
.store = _store, \
|
|
|
|
};
|
|
|
|
|
|
|
|
/* default Control file */
|
2007-07-19 16:49:58 +08:00
|
|
|
MCIDEV_ATTR(reset_counters, S_IWUSR, NULL, mci_reset_counters_store);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* default Attribute files */
|
2007-07-19 16:49:58 +08:00
|
|
|
MCIDEV_ATTR(mc_name, S_IRUGO, mci_ctl_name_show, NULL);
|
|
|
|
MCIDEV_ATTR(size_mb, S_IRUGO, mci_size_mb_show, NULL);
|
|
|
|
MCIDEV_ATTR(seconds_since_reset, S_IRUGO, mci_seconds_show, NULL);
|
|
|
|
MCIDEV_ATTR(ue_noinfo_count, S_IRUGO, mci_ue_noinfo_show, NULL);
|
|
|
|
MCIDEV_ATTR(ce_noinfo_count, S_IRUGO, mci_ce_noinfo_show, NULL);
|
|
|
|
MCIDEV_ATTR(ue_count, S_IRUGO, mci_ue_count_show, NULL);
|
|
|
|
MCIDEV_ATTR(ce_count, S_IRUGO, mci_ce_count_show, NULL);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
/* memory scrubber attribute file */
|
2007-07-19 16:49:58 +08:00
|
|
|
MCIDEV_ATTR(sdram_scrub_rate, S_IRUGO | S_IWUSR, mci_sdram_scrub_rate_show,
|
2007-07-19 16:50:13 +08:00
|
|
|
mci_sdram_scrub_rate_store);
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2007-07-19 16:50:10 +08:00
|
|
|
static struct mcidev_sysfs_attribute *mci_attr[] = {
|
2007-07-19 16:49:33 +08:00
|
|
|
&mci_attr_reset_counters,
|
|
|
|
&mci_attr_mc_name,
|
|
|
|
&mci_attr_size_mb,
|
|
|
|
&mci_attr_seconds_since_reset,
|
|
|
|
&mci_attr_ue_noinfo_count,
|
|
|
|
&mci_attr_ce_noinfo_count,
|
|
|
|
&mci_attr_ue_count,
|
|
|
|
&mci_attr_ce_count,
|
|
|
|
&mci_attr_sdram_scrub_rate,
|
|
|
|
NULL
|
|
|
|
};
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
/*
|
|
|
|
* Release of a MC controlling instance
|
2007-07-19 16:50:27 +08:00
|
|
|
*
|
|
|
|
* each MC control instance has the following resources upon entry:
|
|
|
|
* a) a ref count on the top memctl kobj
|
|
|
|
* b) a ref count on this module
|
|
|
|
*
|
|
|
|
* this function must decrement those ref counts and then
|
|
|
|
* issue a free on the instance's memory
|
2007-07-19 16:49:33 +08:00
|
|
|
*/
|
2007-07-19 16:50:27 +08:00
|
|
|
static void edac_mci_control_release(struct kobject *kobj)
|
2007-07-19 16:49:33 +08:00
|
|
|
{
|
|
|
|
struct mem_ctl_info *mci;
|
|
|
|
|
|
|
|
mci = to_mci(kobj);
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
debugf0("%s() mci instance idx=%d releasing\n", __func__, mci->mc_idx);
|
|
|
|
|
|
|
|
/* decrement the module ref count */
|
|
|
|
module_put(mci->owner);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct kobj_type ktype_mci = {
|
2007-07-19 16:50:27 +08:00
|
|
|
.release = edac_mci_control_release,
|
2007-07-19 16:49:33 +08:00
|
|
|
.sysfs_ops = &mci_ops,
|
2007-07-19 16:49:58 +08:00
|
|
|
.default_attrs = (struct attribute **)mci_attr,
|
2007-07-19 16:49:33 +08:00
|
|
|
};
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* EDAC memory controller sysfs kset:
|
|
|
|
* /sys/devices/system/edac/mc
|
|
|
|
*/
|
2008-07-25 16:49:11 +08:00
|
|
|
static struct kset *mc_kset;
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
/*
|
|
|
|
* edac_mc_register_sysfs_main_kobj
|
|
|
|
*
|
|
|
|
* setups and registers the main kobject for each mci
|
|
|
|
*/
|
|
|
|
int edac_mc_register_sysfs_main_kobj(struct mem_ctl_info *mci)
|
|
|
|
{
|
|
|
|
struct kobject *kobj_mci;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
|
|
|
kobj_mci = &mci->edac_mci_kobj;
|
|
|
|
|
|
|
|
/* Init the mci's kobject */
|
|
|
|
memset(kobj_mci, 0, sizeof(*kobj_mci));
|
|
|
|
|
|
|
|
/* Record which module 'owns' this control structure
|
|
|
|
* and bump the ref count of the module
|
|
|
|
*/
|
|
|
|
mci->owner = THIS_MODULE;
|
|
|
|
|
|
|
|
/* bump ref count on this module */
|
|
|
|
if (!try_module_get(mci->owner)) {
|
|
|
|
err = -ENODEV;
|
|
|
|
goto fail_out;
|
|
|
|
}
|
|
|
|
|
2007-12-18 03:54:39 +08:00
|
|
|
/* this instance become part of the mc_kset */
|
2008-07-25 16:49:11 +08:00
|
|
|
kobj_mci->kset = mc_kset;
|
2007-12-18 03:54:39 +08:00
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* register the mc<id> kobject to the mc_kset */
|
2007-12-18 03:54:39 +08:00
|
|
|
err = kobject_init_and_add(kobj_mci, &ktype_mci, NULL,
|
|
|
|
"mc%d", mci->mc_idx);
|
2007-07-19 16:50:27 +08:00
|
|
|
if (err) {
|
|
|
|
debugf1("%s()Failed to register '.../edac/mc%d'\n",
|
|
|
|
__func__, mci->mc_idx);
|
|
|
|
goto kobj_reg_fail;
|
|
|
|
}
|
2007-12-18 03:54:39 +08:00
|
|
|
kobject_uevent(kobj_mci, KOBJ_ADD);
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
/* At this point, to 'free' the control struct,
|
|
|
|
* edac_mc_unregister_sysfs_main_kobj() must be used
|
|
|
|
*/
|
|
|
|
|
|
|
|
debugf1("%s() Registered '.../edac/mc%d' kobject\n",
|
|
|
|
__func__, mci->mc_idx);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Error exit stack */
|
|
|
|
|
|
|
|
kobj_reg_fail:
|
|
|
|
module_put(mci->owner);
|
|
|
|
|
|
|
|
fail_out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* edac_mc_register_sysfs_main_kobj
|
|
|
|
*
|
|
|
|
* tears down and the main mci kobject from the mc_kset
|
|
|
|
*/
|
|
|
|
void edac_mc_unregister_sysfs_main_kobj(struct mem_ctl_info *mci)
|
|
|
|
{
|
2010-08-17 05:22:43 +08:00
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* delete the kobj from the mc_kset */
|
2007-12-21 00:13:05 +08:00
|
|
|
kobject_put(&mci->edac_mci_kobj);
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
#define EDAC_DEVICE_SYMLINK "device"
|
|
|
|
|
2009-09-26 00:42:25 +08:00
|
|
|
#define grp_to_mci(k) (container_of(k, struct mcidev_sysfs_group_kobj, kobj)->mci)
|
2009-09-25 03:23:42 +08:00
|
|
|
|
|
|
|
/* MCI show/store functions for top most object */
|
|
|
|
static ssize_t inst_grp_show(struct kobject *kobj, struct attribute *attr,
|
|
|
|
char *buffer)
|
|
|
|
{
|
|
|
|
struct mem_ctl_info *mem_ctl_info = grp_to_mci(kobj);
|
|
|
|
struct mcidev_sysfs_attribute *mcidev_attr = to_mcidev_attr(attr);
|
|
|
|
|
|
|
|
debugf1("%s() mem_ctl_info %p\n", __func__, mem_ctl_info);
|
|
|
|
|
|
|
|
if (mcidev_attr->show)
|
|
|
|
return mcidev_attr->show(mem_ctl_info, buffer);
|
|
|
|
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t inst_grp_store(struct kobject *kobj, struct attribute *attr,
|
|
|
|
const char *buffer, size_t count)
|
|
|
|
{
|
|
|
|
struct mem_ctl_info *mem_ctl_info = grp_to_mci(kobj);
|
|
|
|
struct mcidev_sysfs_attribute *mcidev_attr = to_mcidev_attr(attr);
|
|
|
|
|
|
|
|
debugf1("%s() mem_ctl_info %p\n", __func__, mem_ctl_info);
|
|
|
|
|
|
|
|
if (mcidev_attr->store)
|
|
|
|
return mcidev_attr->store(mem_ctl_info, buffer, count);
|
|
|
|
|
|
|
|
return -EIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* No memory to release for this kobj */
|
|
|
|
static void edac_inst_grp_release(struct kobject *kobj)
|
|
|
|
{
|
2009-09-26 00:42:25 +08:00
|
|
|
struct mcidev_sysfs_group_kobj *grp;
|
2009-09-25 03:23:42 +08:00
|
|
|
struct mem_ctl_info *mci;
|
|
|
|
|
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
2009-09-26 00:42:25 +08:00
|
|
|
grp = container_of(kobj, struct mcidev_sysfs_group_kobj, kobj);
|
2009-09-25 03:23:42 +08:00
|
|
|
mci = grp->mci;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Intermediate show/store table */
|
|
|
|
static struct sysfs_ops inst_grp_ops = {
|
|
|
|
.show = inst_grp_show,
|
|
|
|
.store = inst_grp_store
|
|
|
|
};
|
|
|
|
|
|
|
|
/* the kobj_type instance for a instance group */
|
|
|
|
static struct kobj_type ktype_inst_grp = {
|
|
|
|
.release = edac_inst_grp_release,
|
|
|
|
.sysfs_ops = &inst_grp_ops,
|
|
|
|
};
|
|
|
|
|
|
|
|
|
2007-07-19 16:50:10 +08:00
|
|
|
/*
|
2007-07-19 16:50:27 +08:00
|
|
|
* edac_create_mci_instance_attributes
|
2009-09-24 03:26:09 +08:00
|
|
|
* create MC driver specific attributes bellow an specified kobj
|
|
|
|
* This routine calls itself recursively, in order to create an entire
|
|
|
|
* object tree.
|
2007-07-19 16:50:10 +08:00
|
|
|
*/
|
2009-09-25 03:23:42 +08:00
|
|
|
static int edac_create_mci_instance_attributes(struct mem_ctl_info *mci,
|
2010-08-11 05:57:01 +08:00
|
|
|
const struct mcidev_sysfs_attribute *sysfs_attrib,
|
2009-09-24 03:26:09 +08:00
|
|
|
struct kobject *kobj)
|
2007-07-19 16:50:10 +08:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s()\n", __func__);
|
2009-09-25 03:23:42 +08:00
|
|
|
|
2009-09-24 03:26:09 +08:00
|
|
|
while (sysfs_attrib) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() sysfs_attrib = %p\n",__func__, sysfs_attrib);
|
2009-09-24 03:26:09 +08:00
|
|
|
if (sysfs_attrib->grp) {
|
2009-09-26 00:42:25 +08:00
|
|
|
struct mcidev_sysfs_group_kobj *grp_kobj;
|
|
|
|
|
|
|
|
grp_kobj = kzalloc(sizeof(*grp_kobj), GFP_KERNEL);
|
|
|
|
if (!grp_kobj)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
grp_kobj->grp = sysfs_attrib->grp;
|
|
|
|
grp_kobj->mci = mci;
|
2010-08-12 11:30:25 +08:00
|
|
|
list_add_tail(&grp_kobj->list, &mci->grp_kobj_list);
|
2009-09-25 03:36:32 +08:00
|
|
|
|
2009-09-25 03:23:42 +08:00
|
|
|
debugf0("%s() grp %s, mci %p\n", __func__,
|
|
|
|
sysfs_attrib->grp->name, mci);
|
|
|
|
|
2009-09-26 00:42:25 +08:00
|
|
|
err = kobject_init_and_add(&grp_kobj->kobj,
|
|
|
|
&ktype_inst_grp,
|
2009-09-25 03:36:32 +08:00
|
|
|
&mci->edac_mci_kobj,
|
2009-09-24 03:26:09 +08:00
|
|
|
sysfs_attrib->grp->name);
|
2010-08-12 11:30:25 +08:00
|
|
|
if (err < 0) {
|
|
|
|
printk(KERN_ERR "kobject_init_and_add failed: %d\n", err);
|
2009-09-24 03:26:09 +08:00
|
|
|
return err;
|
2010-08-12 11:30:25 +08:00
|
|
|
}
|
2009-09-25 03:23:42 +08:00
|
|
|
err = edac_create_mci_instance_attributes(mci,
|
2009-09-26 00:42:25 +08:00
|
|
|
grp_kobj->grp->mcidev_attr,
|
|
|
|
&grp_kobj->kobj);
|
2009-09-25 03:23:42 +08:00
|
|
|
|
2010-08-12 11:30:25 +08:00
|
|
|
if (err < 0)
|
2009-09-24 03:26:09 +08:00
|
|
|
return err;
|
|
|
|
} else if (sysfs_attrib->attr.name) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() file %s\n", __func__,
|
2009-09-24 03:26:09 +08:00
|
|
|
sysfs_attrib->attr.name);
|
|
|
|
|
|
|
|
err = sysfs_create_file(kobj, &sysfs_attrib->attr);
|
2010-08-12 11:30:25 +08:00
|
|
|
if (err < 0) {
|
|
|
|
printk(KERN_ERR "sysfs_create_file failed: %d\n", err);
|
|
|
|
return err;
|
|
|
|
}
|
2009-09-24 03:26:09 +08:00
|
|
|
} else
|
|
|
|
break;
|
2007-07-19 16:50:10 +08:00
|
|
|
|
|
|
|
sysfs_attrib++;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/*
|
|
|
|
* edac_remove_mci_instance_attributes
|
|
|
|
* remove MC driver specific attributes at the topmost level
|
|
|
|
* directory of this mci instance.
|
|
|
|
*/
|
2009-09-26 00:42:25 +08:00
|
|
|
static void edac_remove_mci_instance_attributes(struct mem_ctl_info *mci,
|
2010-08-11 05:57:01 +08:00
|
|
|
const struct mcidev_sysfs_attribute *sysfs_attrib,
|
2009-09-26 00:42:25 +08:00
|
|
|
struct kobject *kobj, int count)
|
2007-07-19 16:50:27 +08:00
|
|
|
{
|
2009-09-26 00:42:25 +08:00
|
|
|
struct mcidev_sysfs_group_kobj *grp_kobj, *tmp;
|
|
|
|
|
2009-09-25 03:23:42 +08:00
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
2009-09-26 00:42:25 +08:00
|
|
|
/*
|
|
|
|
* loop if there are attributes and until we hit a NULL entry
|
2011-03-31 09:57:33 +08:00
|
|
|
* Remove first all the attributes
|
2009-09-26 00:42:25 +08:00
|
|
|
*/
|
2009-09-24 03:26:09 +08:00
|
|
|
while (sysfs_attrib) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() sysfs_attrib = %p\n",__func__, sysfs_attrib);
|
2009-09-24 03:26:09 +08:00
|
|
|
if (sysfs_attrib->grp) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() seeking for group %s\n",
|
2010-08-12 11:30:25 +08:00
|
|
|
__func__, sysfs_attrib->grp->name);
|
|
|
|
list_for_each_entry(grp_kobj,
|
|
|
|
&mci->grp_kobj_list, list) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() grp_kobj->grp = %p\n",__func__, grp_kobj->grp);
|
2010-08-12 11:30:25 +08:00
|
|
|
if (grp_kobj->grp == sysfs_attrib->grp) {
|
2009-09-26 00:42:25 +08:00
|
|
|
edac_remove_mci_instance_attributes(mci,
|
|
|
|
grp_kobj->grp->mcidev_attr,
|
|
|
|
&grp_kobj->kobj, count + 1);
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() group %s\n", __func__,
|
2010-08-12 11:30:25 +08:00
|
|
|
sysfs_attrib->grp->name);
|
|
|
|
kobject_put(&grp_kobj->kobj);
|
|
|
|
}
|
|
|
|
}
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() end of seeking for group %s\n",
|
2010-08-12 11:30:25 +08:00
|
|
|
__func__, sysfs_attrib->grp->name);
|
2009-09-26 00:42:25 +08:00
|
|
|
} else if (sysfs_attrib->attr.name) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() file %s\n", __func__,
|
2009-09-24 03:26:09 +08:00
|
|
|
sysfs_attrib->attr.name);
|
|
|
|
sysfs_remove_file(kobj, &sysfs_attrib->attr);
|
|
|
|
} else
|
|
|
|
break;
|
2007-07-19 16:50:27 +08:00
|
|
|
sysfs_attrib++;
|
|
|
|
}
|
2009-09-26 00:42:25 +08:00
|
|
|
|
2010-08-12 11:30:25 +08:00
|
|
|
/* Remove the group objects */
|
|
|
|
if (count)
|
|
|
|
return;
|
|
|
|
list_for_each_entry_safe(grp_kobj, tmp,
|
|
|
|
&mci->grp_kobj_list, list) {
|
|
|
|
list_del(&grp_kobj->list);
|
|
|
|
kfree(grp_kobj);
|
|
|
|
}
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
/*
|
|
|
|
* Create a new Memory Controller kobject instance,
|
|
|
|
* mc<id> under the 'mc' directory
|
|
|
|
*
|
|
|
|
* Return:
|
|
|
|
* 0 Success
|
|
|
|
* !0 Failure
|
|
|
|
*/
|
|
|
|
int edac_create_sysfs_mci_device(struct mem_ctl_info *mci)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
int err;
|
|
|
|
struct csrow_info *csrow;
|
2007-07-19 16:50:27 +08:00
|
|
|
struct kobject *kobj_mci = &mci->edac_mci_kobj;
|
2007-07-19 16:49:33 +08:00
|
|
|
|
|
|
|
debugf0("%s() idx=%d\n", __func__, mci->mc_idx);
|
|
|
|
|
2009-09-26 00:42:25 +08:00
|
|
|
INIT_LIST_HEAD(&mci->grp_kobj_list);
|
|
|
|
|
2007-07-19 16:49:33 +08:00
|
|
|
/* create a symlink for the device */
|
2007-07-19 16:50:27 +08:00
|
|
|
err = sysfs_create_link(kobj_mci, &mci->dev->kobj,
|
2007-07-19 16:49:33 +08:00
|
|
|
EDAC_DEVICE_SYMLINK);
|
2007-07-19 16:50:27 +08:00
|
|
|
if (err) {
|
|
|
|
debugf1("%s() failure to create symlink\n", __func__);
|
2007-07-19 16:49:33 +08:00
|
|
|
goto fail0;
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
2007-07-19 16:49:33 +08:00
|
|
|
|
2007-07-19 16:50:10 +08:00
|
|
|
/* If the low level driver desires some attributes,
|
|
|
|
* then create them now for the driver.
|
|
|
|
*/
|
|
|
|
if (mci->mc_driver_sysfs_attributes) {
|
2009-09-25 03:23:42 +08:00
|
|
|
err = edac_create_mci_instance_attributes(mci,
|
2009-09-24 03:26:09 +08:00
|
|
|
mci->mc_driver_sysfs_attributes,
|
|
|
|
&mci->edac_mci_kobj);
|
2007-07-19 16:50:27 +08:00
|
|
|
if (err) {
|
|
|
|
debugf1("%s() failure to create mci attributes\n",
|
|
|
|
__func__);
|
2007-07-19 16:50:10 +08:00
|
|
|
goto fail0;
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
2007-07-19 16:50:10 +08:00
|
|
|
}
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* Make directories for each CSROW object under the mc<id> kobject
|
2007-07-19 16:49:33 +08:00
|
|
|
*/
|
|
|
|
for (i = 0; i < mci->nr_csrows; i++) {
|
|
|
|
csrow = &mci->csrows[i];
|
|
|
|
|
|
|
|
/* Only expose populated CSROWs */
|
|
|
|
if (csrow->nr_pages > 0) {
|
2007-07-19 16:50:27 +08:00
|
|
|
err = edac_create_csrow_object(mci, csrow, i);
|
|
|
|
if (err) {
|
|
|
|
debugf1("%s() failure: create csrow %d obj\n",
|
|
|
|
__func__, i);
|
2007-07-19 16:49:33 +08:00
|
|
|
goto fail1;
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* CSROW error: backout what has already been registered, */
|
2007-07-19 16:50:13 +08:00
|
|
|
fail1:
|
2007-07-19 16:49:58 +08:00
|
|
|
for (i--; i >= 0; i--) {
|
2007-07-19 16:49:33 +08:00
|
|
|
if (csrow->nr_pages > 0) {
|
2007-12-21 00:13:05 +08:00
|
|
|
kobject_put(&mci->csrows[i].kobj);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2007-07-19 16:50:27 +08:00
|
|
|
/* remove the mci instance's attributes, if any */
|
2009-09-26 00:42:25 +08:00
|
|
|
edac_remove_mci_instance_attributes(mci,
|
|
|
|
mci->mc_driver_sysfs_attributes, &mci->edac_mci_kobj, 0);
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
/* remove the symlink */
|
|
|
|
sysfs_remove_link(kobj_mci, EDAC_DEVICE_SYMLINK);
|
|
|
|
|
2007-07-19 16:50:13 +08:00
|
|
|
fail0:
|
2007-07-19 16:49:33 +08:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* remove a Memory Controller instance
|
|
|
|
*/
|
|
|
|
void edac_remove_sysfs_mci_device(struct mem_ctl_info *mci)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
debugf0("%s()\n", __func__);
|
|
|
|
|
|
|
|
/* remove all csrow kobjects */
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() unregister this mci kobj\n", __func__);
|
2007-07-19 16:49:33 +08:00
|
|
|
for (i = 0; i < mci->nr_csrows; i++) {
|
|
|
|
if (mci->csrows[i].nr_pages > 0) {
|
2007-07-19 16:50:27 +08:00
|
|
|
debugf0("%s() unreg csrow-%d\n", __func__, i);
|
2007-12-21 00:13:05 +08:00
|
|
|
kobject_put(&mci->csrows[i].kobj);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2010-08-12 11:30:25 +08:00
|
|
|
/* remove this mci instance's attribtes */
|
|
|
|
if (mci->mc_driver_sysfs_attributes) {
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() unregister mci private attributes\n", __func__);
|
2010-08-12 11:30:25 +08:00
|
|
|
edac_remove_mci_instance_attributes(mci,
|
|
|
|
mci->mc_driver_sysfs_attributes,
|
|
|
|
&mci->edac_mci_kobj, 0);
|
|
|
|
}
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
/* remove the symlink */
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() remove_link\n", __func__);
|
2007-07-19 16:49:33 +08:00
|
|
|
sysfs_remove_link(&mci->edac_mci_kobj, EDAC_DEVICE_SYMLINK);
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
/* unregister this instance's kobject */
|
2011-01-14 02:03:28 +08:00
|
|
|
debugf4("%s() remove_mci_instance\n", __func__);
|
2007-12-21 00:13:05 +08:00
|
|
|
kobject_put(&mci->edac_mci_kobj);
|
2007-07-19 16:49:33 +08:00
|
|
|
}
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
* edac_setup_sysfs_mc_kset(void)
|
|
|
|
*
|
|
|
|
* Initialize the mc_kset for the 'mc' entry
|
|
|
|
* This requires creating the top 'mc' directory with a kset
|
|
|
|
* and its controls/attributes.
|
|
|
|
*
|
|
|
|
* To this 'mc' kset, instance 'mci' will be grouped as children.
|
|
|
|
*
|
|
|
|
* Return: 0 SUCCESS
|
|
|
|
* !0 FAILURE error code
|
|
|
|
*/
|
|
|
|
int edac_sysfs_setup_mc_kset(void)
|
|
|
|
{
|
2010-09-01 22:32:20 +08:00
|
|
|
int err = -EINVAL;
|
2007-07-19 16:50:27 +08:00
|
|
|
struct sysdev_class *edac_class;
|
|
|
|
|
|
|
|
debugf1("%s()\n", __func__);
|
|
|
|
|
|
|
|
/* get the /sys/devices/system/edac class reference */
|
2010-09-02 23:26:48 +08:00
|
|
|
edac_class = edac_get_sysfs_class();
|
2007-07-19 16:50:27 +08:00
|
|
|
if (edac_class == NULL) {
|
|
|
|
debugf1("%s() no edac_class error=%d\n", __func__, err);
|
|
|
|
goto fail_out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Init the MC's kobject */
|
2008-07-25 16:49:11 +08:00
|
|
|
mc_kset = kset_create_and_add("mc", NULL, &edac_class->kset.kobj);
|
|
|
|
if (!mc_kset) {
|
|
|
|
err = -ENOMEM;
|
2007-07-19 16:50:27 +08:00
|
|
|
debugf1("%s() Failed to register '.../edac/mc'\n", __func__);
|
2010-09-02 23:26:48 +08:00
|
|
|
goto fail_kset;
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
|
|
|
|
|
|
|
debugf1("%s() Registered '.../edac/mc' kobject\n", __func__);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
2010-09-02 23:26:48 +08:00
|
|
|
fail_kset:
|
|
|
|
edac_put_sysfs_class();
|
2007-07-19 16:50:27 +08:00
|
|
|
|
|
|
|
fail_out:
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* edac_sysfs_teardown_mc_kset
|
|
|
|
*
|
|
|
|
* deconstruct the mc_ket for memory controllers
|
|
|
|
*/
|
|
|
|
void edac_sysfs_teardown_mc_kset(void)
|
|
|
|
{
|
2008-07-25 16:49:11 +08:00
|
|
|
kset_unregister(mc_kset);
|
2010-09-02 23:26:48 +08:00
|
|
|
edac_put_sysfs_class();
|
2007-07-19 16:50:27 +08:00
|
|
|
}
|
|
|
|
|