linux-sg2042/lib/vsprintf.c

2729 lines
63 KiB
C
Raw Normal View History

/*
* linux/lib/vsprintf.c
*
* Copyright (C) 1991, 1992 Linus Torvalds
*/
/* vsprintf.c -- Lars Wirzenius & Linus Torvalds. */
/*
* Wirzenius wrote this portably, Torvalds fucked it up :-)
*/
/*
* Fri Jul 13 2001 Crutcher Dunnavant <crutcher+kernel@datastacks.com>
* - changed to provide snprintf and vsnprintf functions
* So Feb 1 16:51:32 CET 2004 Juergen Quade <quade@hsnr.de>
* - scnprintf and vscnprintf
*/
#include <stdarg.h>
#include <linux/clk.h>
#include <linux/clk-provider.h>
#include <linux/module.h> /* for KSYM_SYMBOL_LEN */
#include <linux/types.h>
#include <linux/string.h>
#include <linux/ctype.h>
#include <linux/kernel.h>
#include <linux/kallsyms.h>
#include <linux/math64.h>
#include <linux/uaccess.h>
#include <linux/ioport.h>
#include <linux/dcache.h>
vsprintf: check real user/group id for %pK Some setuid binaries will allow reading of files which have read permission by the real user id. This is problematic with files which use %pK because the file access permission is checked at open() time, but the kptr_restrict setting is checked at read() time. If a setuid binary opens a %pK file as an unprivileged user, and then elevates permissions before reading the file, then kernel pointer values may be leaked. This happens for example with the setuid pppd application on Ubuntu 12.04: $ head -1 /proc/kallsyms 00000000 T startup_32 $ pppd file /proc/kallsyms pppd: In file /proc/kallsyms: unrecognized option 'c1000000' This will only leak the pointer value from the first line, but other setuid binaries may leak more information. Fix this by adding a check that in addition to the current process having CAP_SYSLOG, that effective user and group ids are equal to the real ids. If a setuid binary reads the contents of a file which uses %pK then the pointer values will be printed as NULL if the real user is unprivileged. Update the sysctl documentation to reflect the changes, and also correct the documentation to state the kptr_restrict=0 is the default. This is a only temporary solution to the issue. The correct solution is to do the permission check at open() time on files, and to replace %pK with a function which checks the open() time permission. %pK uses in printk should be removed since no sane permission check can be done, and instead protected by using dmesg_restrict. Signed-off-by: Ryan Mallon <rmallon@gmail.com> Cc: Kees Cook <keescook@chromium.org> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Joe Perches <joe@perches.com> Cc: "Eric W. Biederman" <ebiederm@xmission.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-13 07:08:51 +08:00
#include <linux/cred.h>
#include <net/addrconf.h>
#ifdef CONFIG_BLOCK
#include <linux/blkdev.h>
#endif
#include <asm/page.h> /* for PAGE_SIZE */
#include <asm/sections.h> /* for dereference_function_descriptor() */
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
#include <asm/byteorder.h> /* cpu_to_le16 */
#include <linux/string_helpers.h>
#include "kstrtox.h"
/**
* simple_strtoull - convert a string to an unsigned long long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function is obsolete. Please use kstrtoull instead.
*/
unsigned long long simple_strtoull(const char *cp, char **endp, unsigned int base)
{
unsigned long long result;
unsigned int rv;
cp = _parse_integer_fixup_radix(cp, &base);
rv = _parse_integer(cp, base, &result);
/* FIXME */
cp += (rv & ~KSTRTOX_OVERFLOW);
if (endp)
*endp = (char *)cp;
return result;
}
EXPORT_SYMBOL(simple_strtoull);
/**
* simple_strtoul - convert a string to an unsigned long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function is obsolete. Please use kstrtoul instead.
*/
unsigned long simple_strtoul(const char *cp, char **endp, unsigned int base)
{
return simple_strtoull(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtoul);
/**
* simple_strtol - convert a string to a signed long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function is obsolete. Please use kstrtol instead.
*/
long simple_strtol(const char *cp, char **endp, unsigned int base)
{
if (*cp == '-')
return -simple_strtoul(cp + 1, endp, base);
return simple_strtoul(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtol);
/**
* simple_strtoll - convert a string to a signed long long
* @cp: The start of the string
* @endp: A pointer to the end of the parsed string will be placed here
* @base: The number base to use
*
* This function is obsolete. Please use kstrtoll instead.
*/
long long simple_strtoll(const char *cp, char **endp, unsigned int base)
{
if (*cp == '-')
return -simple_strtoull(cp + 1, endp, base);
return simple_strtoull(cp, endp, base);
}
EXPORT_SYMBOL(simple_strtoll);
static noinline_for_stack
int skip_atoi(const char **s)
{
int i = 0;
do {
i = i*10 + *((*s)++) - '0';
} while (isdigit(**s));
return i;
}
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/*
* Decimal conversion is by far the most typical, and is used for
* /proc and /sys data. This directly impacts e.g. top performance
* with many processes running. We optimize it for speed by emitting
* two characters at a time, using a 200 byte lookup table. This
* roughly halves the number of multiplications compared to computing
* the digits one at a time. Implementation strongly inspired by the
* previous version, which in turn used ideas described at
* <http://www.cs.uiowa.edu/~jones/bcd/divide.html> (with permission
* from the author, Douglas W. Jones).
*
* It turns out there is precisely one 26 bit fixed-point
* approximation a of 64/100 for which x/100 == (x * (u64)a) >> 32
* holds for all x in [0, 10^8-1], namely a = 0x28f5c29. The actual
* range happens to be somewhat larger (x <= 1073741898), but that's
* irrelevant for our purpose.
*
* For dividing a number in the range [10^4, 10^6-1] by 100, we still
* need a 32x32->64 bit multiply, so we simply use the same constant.
*
* For dividing a number in the range [100, 10^4-1] by 100, there are
* several options. The simplest is (x * 0x147b) >> 19, which is valid
* for all x <= 43698.
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
*/
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
static const u16 decpair[100] = {
#define _(x) (__force u16) cpu_to_le16(((x % 10) | ((x / 10) << 8)) + 0x3030)
_( 0), _( 1), _( 2), _( 3), _( 4), _( 5), _( 6), _( 7), _( 8), _( 9),
_(10), _(11), _(12), _(13), _(14), _(15), _(16), _(17), _(18), _(19),
_(20), _(21), _(22), _(23), _(24), _(25), _(26), _(27), _(28), _(29),
_(30), _(31), _(32), _(33), _(34), _(35), _(36), _(37), _(38), _(39),
_(40), _(41), _(42), _(43), _(44), _(45), _(46), _(47), _(48), _(49),
_(50), _(51), _(52), _(53), _(54), _(55), _(56), _(57), _(58), _(59),
_(60), _(61), _(62), _(63), _(64), _(65), _(66), _(67), _(68), _(69),
_(70), _(71), _(72), _(73), _(74), _(75), _(76), _(77), _(78), _(79),
_(80), _(81), _(82), _(83), _(84), _(85), _(86), _(87), _(88), _(89),
_(90), _(91), _(92), _(93), _(94), _(95), _(96), _(97), _(98), _(99),
#undef _
};
/*
* This will print a single '0' even if r == 0, since we would
* immediately jump to out_r where two 0s would be written but only
* one of them accounted for in buf. This is needed by ip4_string
* below. All other callers pass a non-zero value of r.
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
*/
static noinline_for_stack
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
char *put_dec_trunc8(char *buf, unsigned r)
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
{
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
unsigned q;
/* 1 <= r < 10^8 */
if (r < 100)
goto out_r;
/* 100 <= r < 10^8 */
q = (r * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 1 <= q < 10^6 */
if (q < 100)
goto out_q;
/* 100 <= q < 10^6 */
r = (q * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[q - 100*r];
buf += 2;
/* 1 <= r < 10^4 */
if (r < 100)
goto out_r;
/* 100 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
out_q:
/* 1 <= q < 100 */
r = q;
out_r:
/* 1 <= r < 100 */
*((u16 *)buf) = decpair[r];
buf += r < 10 ? 1 : 2;
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
return buf;
}
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
#if BITS_PER_LONG == 64 && BITS_PER_LONG_LONG == 64
static noinline_for_stack
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
char *put_dec_full8(char *buf, unsigned r)
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
{
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
unsigned q;
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* 0 <= r < 10^8 */
q = (r * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* 0 <= q < 10^6 */
r = (q * (u64)0x28f5c29) >> 32;
*((u16 *)buf) = decpair[q - 100*r];
buf += 2;
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* 0 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* 0 <= q < 100 */
*((u16 *)buf) = decpair[q];
buf += 2;
return buf;
}
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
static noinline_for_stack
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
char *put_dec(char *buf, unsigned long long n)
{
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
if (n >= 100*1000*1000)
buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
/* 1 <= n <= 1.6e11 */
if (n >= 100*1000*1000)
buf = put_dec_full8(buf, do_div(n, 100*1000*1000));
/* 1 <= n < 1e8 */
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
return put_dec_trunc8(buf, n);
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
}
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
#elif BITS_PER_LONG == 32 && BITS_PER_LONG_LONG == 64
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
static void
put_dec_full4(char *buf, unsigned r)
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
{
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
unsigned q;
/* 0 <= r < 10^4 */
q = (r * 0x147b) >> 19;
*((u16 *)buf) = decpair[r - 100*q];
buf += 2;
/* 0 <= q < 100 */
*((u16 *)buf) = decpair[q];
}
/*
* Call put_dec_full4 on x % 10000, return x / 10000.
* The approximation x/10000 == (x * 0x346DC5D7) >> 43
* holds for all x < 1,128,869,999. The largest value this
* helper will ever be asked to convert is 1,125,520,955.
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
* (second call in the put_dec code, assuming n is all-ones).
*/
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
static noinline_for_stack
unsigned put_dec_helper4(char *buf, unsigned x)
{
uint32_t q = (x * (uint64_t)0x346DC5D7) >> 43;
put_dec_full4(buf, x - q * 10000);
return q;
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
}
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
/* Based on code by Douglas W. Jones found at
* <http://www.cs.uiowa.edu/~jones/bcd/decimal.html#sixtyfour>
* (with permission from the author).
* Performs no 64-bit division and hence should be fast on 32-bit machines.
*/
static
char *put_dec(char *buf, unsigned long long n)
{
uint32_t d3, d2, d1, q, h;
if (n < 100*1000*1000)
return put_dec_trunc8(buf, n);
d1 = ((uint32_t)n >> 16); /* implicit "& 0xffff" */
h = (n >> 32);
d2 = (h ) & 0xffff;
d3 = (h >> 16); /* implicit "& 0xffff" */
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* n = 2^48 d3 + 2^32 d2 + 2^16 d1 + d0
= 281_4749_7671_0656 d3 + 42_9496_7296 d2 + 6_5536 d1 + d0 */
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
q = 656 * d3 + 7296 * d2 + 5536 * d1 + ((uint32_t)n & 0xffff);
q = put_dec_helper4(buf, q);
q += 7671 * d3 + 9496 * d2 + 6 * d1;
q = put_dec_helper4(buf+4, q);
q += 4749 * d3 + 42 * d2;
q = put_dec_helper4(buf+8, q);
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
q += 281 * d3;
buf += 12;
if (q)
buf = put_dec_trunc8(buf, q);
else while (buf[-1] == '0')
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
--buf;
return buf;
}
#endif
procfs: add num_to_str() to speed up /proc/stat == stat_check.py num = 0 with open("/proc/stat") as f: while num < 1000 : data = f.read() f.seek(0, 0) num = num + 1 == perf shows 20.39% stat_check.py [kernel.kallsyms] [k] format_decode 13.41% stat_check.py [kernel.kallsyms] [k] number 12.61% stat_check.py [kernel.kallsyms] [k] vsnprintf 10.85% stat_check.py [kernel.kallsyms] [k] memcpy 4.85% stat_check.py [kernel.kallsyms] [k] radix_tree_lookup 4.43% stat_check.py [kernel.kallsyms] [k] seq_printf This patch removes most of calls to vsnprintf() by adding num_to_str() and seq_print_decimal_ull(), which prints decimal numbers without rich functions provided by printf(). On my 8cpu box. == Before patch == [root@bluextal test]# time ./stat_check.py real 0m0.150s user 0m0.026s sys 0m0.121s == After patch == [root@bluextal test]# time ./stat_check.py real 0m0.055s user 0m0.022s sys 0m0.030s [akpm@linux-foundation.org: remove incorrect comment, use less statck in num_to_str(), move comment from .h to .c, simplify seq_put_decimal_ull()] [andrea@betterlinux.com: avoid breaking the ABI in /proc/stat] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Signed-off-by: Andrea Righi <andrea@betterlinux.com> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Glauber Costa <glommer@parallels.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Ingo Molnar <mingo@elte.hu> Cc: Paul Turner <pjt@google.com> Cc: Russell King <rmk@arm.linux.org.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-24 06:02:54 +08:00
/*
* Convert passed number to decimal string.
* Returns the length of string. On buffer overflow, returns 0.
*
* If speed is not important, use snprintf(). It's easy to read the code.
*/
int num_to_str(char *buf, int size, unsigned long long num)
{
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* put_dec requires 2-byte alignment of the buffer. */
char tmp[sizeof(num) * 3] __aligned(2);
procfs: add num_to_str() to speed up /proc/stat == stat_check.py num = 0 with open("/proc/stat") as f: while num < 1000 : data = f.read() f.seek(0, 0) num = num + 1 == perf shows 20.39% stat_check.py [kernel.kallsyms] [k] format_decode 13.41% stat_check.py [kernel.kallsyms] [k] number 12.61% stat_check.py [kernel.kallsyms] [k] vsnprintf 10.85% stat_check.py [kernel.kallsyms] [k] memcpy 4.85% stat_check.py [kernel.kallsyms] [k] radix_tree_lookup 4.43% stat_check.py [kernel.kallsyms] [k] seq_printf This patch removes most of calls to vsnprintf() by adding num_to_str() and seq_print_decimal_ull(), which prints decimal numbers without rich functions provided by printf(). On my 8cpu box. == Before patch == [root@bluextal test]# time ./stat_check.py real 0m0.150s user 0m0.026s sys 0m0.121s == After patch == [root@bluextal test]# time ./stat_check.py real 0m0.055s user 0m0.022s sys 0m0.030s [akpm@linux-foundation.org: remove incorrect comment, use less statck in num_to_str(), move comment from .h to .c, simplify seq_put_decimal_ull()] [andrea@betterlinux.com: avoid breaking the ABI in /proc/stat] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Signed-off-by: Andrea Righi <andrea@betterlinux.com> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Glauber Costa <glommer@parallels.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Ingo Molnar <mingo@elte.hu> Cc: Paul Turner <pjt@google.com> Cc: Russell King <rmk@arm.linux.org.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-24 06:02:54 +08:00
int idx, len;
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
/* put_dec() may work incorrectly for num = 0 (generate "", not "0") */
if (num <= 9) {
tmp[0] = '0' + num;
len = 1;
} else {
len = put_dec(tmp, num) - tmp;
}
procfs: add num_to_str() to speed up /proc/stat == stat_check.py num = 0 with open("/proc/stat") as f: while num < 1000 : data = f.read() f.seek(0, 0) num = num + 1 == perf shows 20.39% stat_check.py [kernel.kallsyms] [k] format_decode 13.41% stat_check.py [kernel.kallsyms] [k] number 12.61% stat_check.py [kernel.kallsyms] [k] vsnprintf 10.85% stat_check.py [kernel.kallsyms] [k] memcpy 4.85% stat_check.py [kernel.kallsyms] [k] radix_tree_lookup 4.43% stat_check.py [kernel.kallsyms] [k] seq_printf This patch removes most of calls to vsnprintf() by adding num_to_str() and seq_print_decimal_ull(), which prints decimal numbers without rich functions provided by printf(). On my 8cpu box. == Before patch == [root@bluextal test]# time ./stat_check.py real 0m0.150s user 0m0.026s sys 0m0.121s == After patch == [root@bluextal test]# time ./stat_check.py real 0m0.055s user 0m0.022s sys 0m0.030s [akpm@linux-foundation.org: remove incorrect comment, use less statck in num_to_str(), move comment from .h to .c, simplify seq_put_decimal_ull()] [andrea@betterlinux.com: avoid breaking the ABI in /proc/stat] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Signed-off-by: Andrea Righi <andrea@betterlinux.com> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Glauber Costa <glommer@parallels.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Ingo Molnar <mingo@elte.hu> Cc: Paul Turner <pjt@google.com> Cc: Russell King <rmk@arm.linux.org.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-24 06:02:54 +08:00
if (len > size)
return 0;
for (idx = 0; idx < len; ++idx)
buf[idx] = tmp[len - idx - 1];
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
return len;
procfs: add num_to_str() to speed up /proc/stat == stat_check.py num = 0 with open("/proc/stat") as f: while num < 1000 : data = f.read() f.seek(0, 0) num = num + 1 == perf shows 20.39% stat_check.py [kernel.kallsyms] [k] format_decode 13.41% stat_check.py [kernel.kallsyms] [k] number 12.61% stat_check.py [kernel.kallsyms] [k] vsnprintf 10.85% stat_check.py [kernel.kallsyms] [k] memcpy 4.85% stat_check.py [kernel.kallsyms] [k] radix_tree_lookup 4.43% stat_check.py [kernel.kallsyms] [k] seq_printf This patch removes most of calls to vsnprintf() by adding num_to_str() and seq_print_decimal_ull(), which prints decimal numbers without rich functions provided by printf(). On my 8cpu box. == Before patch == [root@bluextal test]# time ./stat_check.py real 0m0.150s user 0m0.026s sys 0m0.121s == After patch == [root@bluextal test]# time ./stat_check.py real 0m0.055s user 0m0.022s sys 0m0.030s [akpm@linux-foundation.org: remove incorrect comment, use less statck in num_to_str(), move comment from .h to .c, simplify seq_put_decimal_ull()] [andrea@betterlinux.com: avoid breaking the ABI in /proc/stat] Signed-off-by: KAMEZAWA Hiroyuki <kamezawa.hiroyu@jp.fujitsu.com> Signed-off-by: Andrea Righi <andrea@betterlinux.com> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Glauber Costa <glommer@parallels.com> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Ingo Molnar <mingo@elte.hu> Cc: Paul Turner <pjt@google.com> Cc: Russell King <rmk@arm.linux.org.uk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-03-24 06:02:54 +08:00
}
#define SIGN 1 /* unsigned/signed, must be 1 */
#define LEFT 2 /* left justified */
#define PLUS 4 /* show plus */
#define SPACE 8 /* space if plus */
#define ZEROPAD 16 /* pad with zero, must be 16 == '0' - ' ' */
#define SMALL 32 /* use lowercase in hex (must be 32 == 0x20) */
#define SPECIAL 64 /* prefix hex with "0x", octal with "0" */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
enum format_type {
FORMAT_TYPE_NONE, /* Just a string part */
FORMAT_TYPE_WIDTH,
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
FORMAT_TYPE_PRECISION,
FORMAT_TYPE_CHAR,
FORMAT_TYPE_STR,
FORMAT_TYPE_PTR,
FORMAT_TYPE_PERCENT_CHAR,
FORMAT_TYPE_INVALID,
FORMAT_TYPE_LONG_LONG,
FORMAT_TYPE_ULONG,
FORMAT_TYPE_LONG,
FORMAT_TYPE_UBYTE,
FORMAT_TYPE_BYTE,
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
FORMAT_TYPE_USHORT,
FORMAT_TYPE_SHORT,
FORMAT_TYPE_UINT,
FORMAT_TYPE_INT,
FORMAT_TYPE_SIZE_T,
FORMAT_TYPE_PTRDIFF
};
struct printf_spec {
u8 type; /* format_type enum */
u8 flags; /* flags to number() */
u8 base; /* number base, 8, 10 or 16 only */
u8 qualifier; /* number qualifier, one of 'hHlLtzZ' */
s16 field_width; /* width of output field */
s16 precision; /* # of digits/chars */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
};
static noinline_for_stack
char *number(char *buf, char *end, unsigned long long num,
struct printf_spec spec)
{
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
/* put_dec requires 2-byte alignment of the buffer. */
char tmp[3 * sizeof(num)] __aligned(2);
char sign;
char locase;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
int need_pfx = ((spec.flags & SPECIAL) && spec.base != 10);
int i;
bool is_zero = num == 0LL;
/* locase = 0 or 0x20. ORing digits or letters with 'locase'
* produces same digits or (maybe lowercased) letters */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
locase = (spec.flags & SMALL);
if (spec.flags & LEFT)
spec.flags &= ~ZEROPAD;
sign = 0;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (spec.flags & SIGN) {
if ((signed long long)num < 0) {
sign = '-';
num = -(signed long long)num;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width--;
} else if (spec.flags & PLUS) {
sign = '+';
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width--;
} else if (spec.flags & SPACE) {
sign = ' ';
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width--;
}
}
if (need_pfx) {
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (spec.base == 16)
spec.field_width -= 2;
else if (!is_zero)
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width--;
}
/* generate full string in tmp[], in reverse order */
i = 0;
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
if (num < spec.base)
tmp[i++] = hex_asc_upper[num] | locase;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
else if (spec.base != 10) { /* 8 or 16 */
int mask = spec.base - 1;
int shift = 3;
if (spec.base == 16)
shift = 4;
do {
tmp[i++] = (hex_asc_upper[((unsigned char)num) & mask] | locase);
num >>= shift;
} while (num);
vsprintf.c: optimizing, part 2: base 10 conversion speedup, v2 Optimize integer-to-string conversion in vsprintf.c for base 10. This is by far the most used conversion, and in some use cases it impacts performance. For example, top reads /proc/$PID/stat for every process, and with 4000 processes decimal conversion alone takes noticeable time. Using code from http://www.cs.uiowa.edu/~jones/bcd/decimal.html (with permission from the author, Douglas W. Jones) binary-to-decimal-string conversion is done in groups of five digits at once, using only additions/subtractions/shifts (with -O2; -Os throws in some multiply instructions). On i386 arch gcc 4.1.2 -O2 generates ~500 bytes of code. This patch is run tested. Userspace benchmark/test is also attached. I tested it on PIII and AMD64 and new code is generally ~2.5 times faster. On AMD64: # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 12895992590592 ok... [Ctrl-C] # ./vsprintf_verify-O2 Original decimal conv: .......... 151 ns per iteration Patched decimal conv: .......... 62 ns per iteration Testing correctness 26025406464 ok... [Ctrl-C] More realistic test: top from busybox project was modified to report how many us it took to scan /proc (this does not account any processing done after that, like sorting process list), and then I test it with 4000 processes: #!/bin/sh i=4000 while test $i != 0; do sleep 30 & let i-- done busybox top -b -n3 >/dev/null on unpatched kernel: top: 4120 processes took 102864 microseconds to scan top: 4120 processes took 91757 microseconds to scan top: 4120 processes took 92517 microseconds to scan top: 4120 processes took 92581 microseconds to scan on patched kernel: top: 4120 processes took 75460 microseconds to scan top: 4120 processes took 66451 microseconds to scan top: 4120 processes took 67267 microseconds to scan top: 4120 processes took 67618 microseconds to scan The speedup comes from much faster generation of /proc/PID/stat by sprintf() calls inside the kernel. Signed-off-by: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2007-07-16 14:41:56 +08:00
} else { /* base 10 */
i = put_dec(tmp, num) - tmp;
}
/* printing 100 using %2d gives "100", not "00" */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (i > spec.precision)
spec.precision = i;
/* leading space padding */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width -= spec.precision;
if (!(spec.flags & (ZEROPAD | LEFT))) {
while (--spec.field_width >= 0) {
if (buf < end)
*buf = ' ';
++buf;
}
}
/* sign */
if (sign) {
if (buf < end)
*buf = sign;
++buf;
}
/* "0x" / "0" prefix */
if (need_pfx) {
if (spec.base == 16 || !is_zero) {
if (buf < end)
*buf = '0';
++buf;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (spec.base == 16) {
if (buf < end)
*buf = ('X' | locase);
++buf;
}
}
/* zero or space padding */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (!(spec.flags & LEFT)) {
char c = ' ' + (spec.flags & ZEROPAD);
BUILD_BUG_ON(' ' + ZEROPAD != '0');
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (--spec.field_width >= 0) {
if (buf < end)
*buf = c;
++buf;
}
}
/* hmm even more zero padding? */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (i <= --spec.precision) {
if (buf < end)
*buf = '0';
++buf;
}
/* actual digits of result */
while (--i >= 0) {
if (buf < end)
*buf = tmp[i];
++buf;
}
/* trailing space padding */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (--spec.field_width >= 0) {
if (buf < end)
*buf = ' ';
++buf;
}
return buf;
}
static noinline_for_stack
char *string(char *buf, char *end, const char *s, struct printf_spec spec)
{
int len, i;
if ((unsigned long)s < PAGE_SIZE)
s = "(null)";
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
len = strnlen(s, spec.precision);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (!(spec.flags & LEFT)) {
while (len < spec.field_width--) {
if (buf < end)
*buf = ' ';
++buf;
}
}
for (i = 0; i < len; ++i) {
if (buf < end)
*buf = *s;
++buf; ++s;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (len < spec.field_width--) {
if (buf < end)
*buf = ' ';
++buf;
}
return buf;
}
static void widen(char *buf, char *end, unsigned len, unsigned spaces)
{
size_t size;
if (buf >= end) /* nowhere to put anything */
return;
size = end - buf;
if (size <= spaces) {
memset(buf, ' ', size);
return;
}
if (len) {
if (len > size - spaces)
len = size - spaces;
memmove(buf + spaces, buf, len);
}
memset(buf, ' ', spaces);
}
static noinline_for_stack
char *dentry_name(char *buf, char *end, const struct dentry *d, struct printf_spec spec,
const char *fmt)
{
const char *array[4], *s;
const struct dentry *p;
int depth;
int i, n;
switch (fmt[1]) {
case '2': case '3': case '4':
depth = fmt[1] - '0';
break;
default:
depth = 1;
}
rcu_read_lock();
for (i = 0; i < depth; i++, d = p) {
p = ACCESS_ONCE(d->d_parent);
array[i] = ACCESS_ONCE(d->d_name.name);
if (p == d) {
if (i)
array[i] = "";
i++;
break;
}
}
s = array[--i];
for (n = 0; n != spec.precision; n++, buf++) {
char c = *s++;
if (!c) {
if (!i)
break;
c = '/';
s = array[--i];
}
if (buf < end)
*buf = c;
}
rcu_read_unlock();
if (n < spec.field_width) {
/* we want to pad the sucker */
unsigned spaces = spec.field_width - n;
if (!(spec.flags & LEFT)) {
widen(buf - n, end, n, spaces);
return buf + spaces;
}
while (spaces--) {
if (buf < end)
*buf = ' ';
++buf;
}
}
return buf;
}
#ifdef CONFIG_BLOCK
static noinline_for_stack
char *bdev_name(char *buf, char *end, struct block_device *bdev,
struct printf_spec spec, const char *fmt)
{
struct gendisk *hd = bdev->bd_disk;
buf = string(buf, end, hd->disk_name, spec);
if (bdev->bd_part->partno) {
if (isdigit(hd->disk_name[strlen(hd->disk_name)-1])) {
if (buf < end)
*buf = 'p';
buf++;
}
buf = number(buf, end, bdev->bd_part->partno, spec);
}
return buf;
}
#endif
static noinline_for_stack
char *symbol_string(char *buf, char *end, void *ptr,
struct printf_spec spec, const char *fmt)
{
unsigned long value;
#ifdef CONFIG_KALLSYMS
char sym[KSYM_SYMBOL_LEN];
#endif
if (fmt[1] == 'R')
ptr = __builtin_extract_return_addr(ptr);
value = (unsigned long)ptr;
#ifdef CONFIG_KALLSYMS
if (*fmt == 'B')
sprint_backtrace(sym, value);
else if (*fmt != 'f' && *fmt != 's')
vsprintf: introduce %pf format specifier A printf format specifier which would allow us to print a pure function name has been suggested by Andrew Morton a couple of months ago. The current %pF is very convenient to print a function symbol, but often we only want to print the name of the function, without its asm offset. That's what %pf does in this patch. The lowecase f has been chosen for its intuitive meaning of a 'weak kind of %pF'. The support for this new format would be welcome by the tracing code where the need to print pure function names is often needed. This is also true for other parts of the kernel: $ git-grep -E "kallsyms_lookup\(.+?\)" arch/blackfin/kernel/traps.c: symname = kallsyms_lookup(address, &symsize, &offset, &modname, namebuf); arch/powerpc/xmon/xmon.c: name = kallsyms_lookup(pc, &size, &offset, NULL, tmpstr); arch/sh/kernel/cpu/sh5/unwind.c: sym = kallsyms_lookup(pc, NULL, &offset, NULL, namebuf); arch/x86/kernel/ftrace.c: kallsyms_lookup((unsigned long) syscall, NULL, NULL, NULL, str); kernel/kprobes.c: sym = kallsyms_lookup((unsigned long)p->addr, NULL, kernel/lockdep.c: return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); kernel/trace/ftrace.c: kallsyms_lookup(*ptr, NULL, NULL, NULL, str); kernel/trace/trace_functions.c: kallsyms_lookup(ip, NULL, NULL, NULL, str); kernel/trace/trace_output.c: kallsyms_lookup(address, NULL, NULL, NULL, str); Changes in v2: - Add the explanation of the %pf role for vsnprintf() and bstr_printf() - Change the comments by dropping the "asm offset" notion and only define the %pf against the actual function offset notion. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Acked-by: Mike Frysinger <vapier@gentoo.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Zhaolei <zhaolei@cn.fujitsu.com> Cc: Tom Zanussi <tzanussi@gmail.com> Cc: Li Zefan <lizf@cn.fujitsu.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <20090415154817.GC5989@nowhere> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-04-15 23:48:18 +08:00
sprint_symbol(sym, value);
else
sprint_symbol_no_offset(sym, value);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, sym, spec);
#else
spec.field_width = 2 * sizeof(void *);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.flags |= SPECIAL | SMALL | ZEROPAD;
spec.base = 16;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return number(buf, end, value, spec);
#endif
}
static noinline_for_stack
char *resource_string(char *buf, char *end, struct resource *res,
struct printf_spec spec, const char *fmt)
{
#ifndef IO_RSRC_PRINTK_SIZE
#define IO_RSRC_PRINTK_SIZE 6
#endif
#ifndef MEM_RSRC_PRINTK_SIZE
#define MEM_RSRC_PRINTK_SIZE 10
#endif
static const struct printf_spec io_spec = {
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
.base = 16,
.field_width = IO_RSRC_PRINTK_SIZE,
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
.precision = -1,
.flags = SPECIAL | SMALL | ZEROPAD,
};
static const struct printf_spec mem_spec = {
.base = 16,
.field_width = MEM_RSRC_PRINTK_SIZE,
.precision = -1,
.flags = SPECIAL | SMALL | ZEROPAD,
};
static const struct printf_spec bus_spec = {
.base = 16,
.field_width = 2,
.precision = -1,
.flags = SMALL | ZEROPAD,
};
static const struct printf_spec dec_spec = {
.base = 10,
.precision = -1,
.flags = 0,
};
static const struct printf_spec str_spec = {
.field_width = -1,
.precision = 10,
.flags = LEFT,
};
static const struct printf_spec flag_spec = {
.base = 16,
.precision = -1,
.flags = SPECIAL | SMALL,
};
/* 32-bit res (sizeof==4): 10 chars in dec, 10 in hex ("0x" + 8)
* 64-bit res (sizeof==8): 20 chars in dec, 18 in hex ("0x" + 16) */
#define RSRC_BUF_SIZE ((2 * sizeof(resource_size_t)) + 4)
#define FLAG_BUF_SIZE (2 * sizeof(res->flags))
#define DECODED_BUF_SIZE sizeof("[mem - 64bit pref window disabled]")
#define RAW_BUF_SIZE sizeof("[mem - flags 0x]")
char sym[max(2*RSRC_BUF_SIZE + DECODED_BUF_SIZE,
2*RSRC_BUF_SIZE + FLAG_BUF_SIZE + RAW_BUF_SIZE)];
char *p = sym, *pend = sym + sizeof(sym);
int decode = (fmt[0] == 'R') ? 1 : 0;
const struct printf_spec *specp;
*p++ = '[';
if (res->flags & IORESOURCE_IO) {
p = string(p, pend, "io ", str_spec);
specp = &io_spec;
} else if (res->flags & IORESOURCE_MEM) {
p = string(p, pend, "mem ", str_spec);
specp = &mem_spec;
} else if (res->flags & IORESOURCE_IRQ) {
p = string(p, pend, "irq ", str_spec);
specp = &dec_spec;
} else if (res->flags & IORESOURCE_DMA) {
p = string(p, pend, "dma ", str_spec);
specp = &dec_spec;
} else if (res->flags & IORESOURCE_BUS) {
p = string(p, pend, "bus ", str_spec);
specp = &bus_spec;
} else {
p = string(p, pend, "??? ", str_spec);
specp = &mem_spec;
decode = 0;
}
if (decode && res->flags & IORESOURCE_UNSET) {
p = string(p, pend, "size ", str_spec);
p = number(p, pend, resource_size(res), *specp);
} else {
p = number(p, pend, res->start, *specp);
if (res->start != res->end) {
*p++ = '-';
p = number(p, pend, res->end, *specp);
}
}
if (decode) {
if (res->flags & IORESOURCE_MEM_64)
p = string(p, pend, " 64bit", str_spec);
if (res->flags & IORESOURCE_PREFETCH)
p = string(p, pend, " pref", str_spec);
if (res->flags & IORESOURCE_WINDOW)
p = string(p, pend, " window", str_spec);
if (res->flags & IORESOURCE_DISABLED)
p = string(p, pend, " disabled", str_spec);
} else {
p = string(p, pend, " flags ", str_spec);
p = number(p, pend, res->flags, flag_spec);
}
*p++ = ']';
*p = '\0';
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, sym, spec);
}
static noinline_for_stack
char *hex_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
const char *fmt)
{
int i, len = 1; /* if we pass '%ph[CDN]', field width remains
negative value, fallback to the default */
char separator;
if (spec.field_width == 0)
/* nothing to print */
return buf;
if (ZERO_OR_NULL_PTR(addr))
/* NULL pointer */
return string(buf, end, NULL, spec);
switch (fmt[1]) {
case 'C':
separator = ':';
break;
case 'D':
separator = '-';
break;
case 'N':
separator = 0;
break;
default:
separator = ' ';
break;
}
if (spec.field_width > 0)
len = min_t(int, spec.field_width, 64);
for (i = 0; i < len; ++i) {
if (buf < end)
*buf = hex_asc_hi(addr[i]);
++buf;
if (buf < end)
*buf = hex_asc_lo(addr[i]);
++buf;
if (separator && i != len - 1) {
if (buf < end)
*buf = separator;
++buf;
}
}
return buf;
}
lib/vsprintf: implement bitmap printing through '%*pb[l]' bitmap and its derivatives such as cpumask and nodemask currently only provide formatting functions which put the output string into the provided buffer; however, how long this buffer should be isn't defined anywhere and given that some of these bitmaps can be too large to be formatted into an on-stack buffer it users sometimes are unnecessarily forced to come up with creative solutions and compromises for the buffer just to printk these bitmaps. There have been a couple different attempts at making this easier. 1. Way back, PeterZ tried printk '%pb' extension with the precision for bit width - '%.*pb'. This was intuitive and made sense but unfortunately triggered a compile warning about using precision for a pointer. http://lkml.kernel.org/g/1336577562.2527.58.camel@twins 2. I implemented bitmap_pr_cont[_list]() and its wrappers for cpumask and nodemask. This works but PeterZ pointed out that pr_cont's tendency to produce broken lines when multiple CPUs are printing is bothering considering the usages. http://lkml.kernel.org/g/1418226774-30215-3-git-send-email-tj@kernel.org So, this patch is another attempt at teaching printk and friends how to print bitmaps. It's almost identical to what PeterZ tried with precision but it uses the field width for the number of bits instead of precision. The format used is '%*pb[l]', with the optional trailing 'l' specifying list format instead of hex masks. This is a valid format string and doesn't trigger compiler warnings; however, it does make it impossible to specify output field width when printing bitmaps. I think this is an acceptable trade-off given how much easier it makes printing bitmaps and that we don't have any in-kernel user which is using the field width specification. If any future user wants to use field width with a bitmap, it'd have to format the bitmap into a string buffer and then print that buffer with width spec, which isn't different from how it should be done now. This patch implements bitmap[_list]_string() which are called from the vsprintf pointer() formatting function. The implementation is mostly identical to bitmap_scn[list]printf() except that the output is performed in the vsprintf way. These functions handle formatting into too small buffers and sprintf() family of functions report the correct overrun output length. bitmap_scn[list]printf() are now thin wrappers around scnprintf(). Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com> Cc: "John W. Linville" <linville@tuxdriver.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Chris Metcalf <cmetcalf@tilera.com> Cc: Chris Zankel <chris@zankel.net> Cc: Christoph Lameter <cl@linux.com> Cc: Dmitry Torokhov <dmitry.torokhov@gmail.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Li Zefan <lizefan@huawei.com> Cc: Max Filippov <jcmvbkbc@gmail.com> Cc: Mike Travis <travis@sgi.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Steffen Klassert <steffen.klassert@secunet.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-14 06:36:53 +08:00
static noinline_for_stack
char *bitmap_string(char *buf, char *end, unsigned long *bitmap,
struct printf_spec spec, const char *fmt)
{
const int CHUNKSZ = 32;
int nr_bits = max_t(int, spec.field_width, 0);
int i, chunksz;
bool first = true;
/* reused to print numbers */
spec = (struct printf_spec){ .flags = SMALL | ZEROPAD, .base = 16 };
chunksz = nr_bits & (CHUNKSZ - 1);
if (chunksz == 0)
chunksz = CHUNKSZ;
i = ALIGN(nr_bits, CHUNKSZ) - CHUNKSZ;
for (; i >= 0; i -= CHUNKSZ) {
u32 chunkmask, val;
int word, bit;
chunkmask = ((1ULL << chunksz) - 1);
word = i / BITS_PER_LONG;
bit = i % BITS_PER_LONG;
val = (bitmap[word] >> bit) & chunkmask;
if (!first) {
if (buf < end)
*buf = ',';
buf++;
}
first = false;
spec.field_width = DIV_ROUND_UP(chunksz, 4);
buf = number(buf, end, val, spec);
chunksz = CHUNKSZ;
}
return buf;
}
static noinline_for_stack
char *bitmap_list_string(char *buf, char *end, unsigned long *bitmap,
struct printf_spec spec, const char *fmt)
{
int nr_bits = max_t(int, spec.field_width, 0);
/* current bit is 'cur', most recently seen range is [rbot, rtop] */
int cur, rbot, rtop;
bool first = true;
/* reused to print numbers */
spec = (struct printf_spec){ .base = 10 };
rbot = cur = find_first_bit(bitmap, nr_bits);
while (cur < nr_bits) {
rtop = cur;
cur = find_next_bit(bitmap, nr_bits, cur + 1);
if (cur < nr_bits && cur <= rtop + 1)
continue;
if (!first) {
if (buf < end)
*buf = ',';
buf++;
}
first = false;
buf = number(buf, end, rbot, spec);
if (rbot < rtop) {
if (buf < end)
*buf = '-';
buf++;
buf = number(buf, end, rtop, spec);
}
rbot = cur;
}
return buf;
}
static noinline_for_stack
char *mac_address_string(char *buf, char *end, u8 *addr,
struct printf_spec spec, const char *fmt)
{
char mac_addr[sizeof("xx:xx:xx:xx:xx:xx")];
char *p = mac_addr;
int i;
char separator;
bool reversed = false;
switch (fmt[1]) {
case 'F':
separator = '-';
break;
case 'R':
reversed = true;
/* fall through */
default:
separator = ':';
break;
}
for (i = 0; i < 6; i++) {
if (reversed)
p = hex_byte_pack(p, addr[5 - i]);
else
p = hex_byte_pack(p, addr[i]);
if (fmt[0] == 'M' && i != 5)
*p++ = separator;
}
*p = '\0';
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, mac_addr, spec);
}
static noinline_for_stack
char *ip4_string(char *p, const u8 *addr, const char *fmt)
{
int i;
bool leading_zeros = (fmt[0] == 'i');
int index;
int step;
switch (fmt[2]) {
case 'h':
#ifdef __BIG_ENDIAN
index = 0;
step = 1;
#else
index = 3;
step = -1;
#endif
break;
case 'l':
index = 3;
step = -1;
break;
case 'n':
case 'b':
default:
index = 0;
step = 1;
break;
}
for (i = 0; i < 4; i++) {
lib/vsprintf.c: even faster binary to decimal conversion The most expensive part of decimal conversion is the divisions by 10 (albeit done using reciprocal multiplication with appropriately chosen constants). I decided to see if one could eliminate around half of these multiplications by emitting two digits at a time, at the cost of a 200 byte lookup table, and it does indeed seem like there is something to be gained, especially on 64 bits. Microbenchmarking shows improvements ranging from -50% (for numbers uniformly distributed in [0, 2^64-1]) to -25% (for numbers heavily biased toward the smaller end, a more realistic distribution). On a larger scale, perf shows that top, one of the big consumers of /proc data, uses 0.5-1.0% fewer cpu cycles. I had to jump through some hoops to get the 32 bit code to compile and run on my 64 bit machine, so I'm not sure how relevant these numbers are, but just for comparison the microbenchmark showed improvements between -30% and -10%. The bloat-o-meter costs are around 150 bytes (the generated code is a little smaller, so it's not the full 200 bytes) on both 32 and 64 bit. I'm aware that extra cache misses won't show up in a microbenchmark as used above, but on the other hand decimal conversions often happen in bulk (for example in the case of top). I have of course tested that the new code generates the same output as the old, for both the first and last 1e10 numbers in [0,2^64-1] and 4e9 'random' numbers in-between. Test and verification code on github: https://github.com/Villemoes/dec. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Tested-by: Jeff Epler <jepler@unpythonic.net> Cc: "Peter Zijlstra (Intel)" <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Joe Perches <joe@perches.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-17 03:43:22 +08:00
char temp[4] __aligned(2); /* hold each IP quad in reverse order */
vsprintf: further optimize decimal conversion Previous code was using optimizations which were developed to work well even on narrow-word CPUs (by today's standards). But Linux runs only on 32-bit and wider CPUs. We can use that. First: using 32x32->64 multiply and trivial 32-bit shift, we can correctly divide by 10 much larger numbers, and thus we can print groups of 9 digits instead of groups of 5 digits. Next: there are two algorithms to print larger numbers. One is generic: divide by 1000000000 and repeatedly print groups of (up to) 9 digits. It's conceptually simple, but requires an (unsigned long long) / 1000000000 division. Second algorithm splits 64-bit unsigned long long into 16-bit chunks, manipulates them cleverly and generates groups of 4 decimal digits. It so happens that it does NOT require long long division. If long is > 32 bits, division of 64-bit values is relatively easy, and we will use the first algorithm. If long long is > 64 bits (strange architecture with VERY large long long), second algorithm can't be used, and we again use the first one. Else (if long is 32 bits and long long is 64 bits) we use second one. And third: there is a simple optimization which takes fast path not only for zero as was done before, but for all one-digit numbers. In all tested cases new code is faster than old one, in many cases by 30%, in few cases by more than 50% (for example, on x86-32, conversion of 12345678). Code growth is ~0 in 32-bit case and ~130 bytes in 64-bit case. This patch is based upon an original from Michal Nazarewicz. [akpm@linux-foundation.org: checkpatch fixes] Signed-off-by: Michal Nazarewicz <mina86@mina86.com> Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com> Cc: Douglas W Jones <jones@cs.uiowa.edu> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2012-06-01 07:26:08 +08:00
int digits = put_dec_trunc8(temp, addr[index]) - temp;
if (leading_zeros) {
if (digits < 3)
*p++ = '0';
if (digits < 2)
*p++ = '0';
}
/* reverse the digits in the quad */
while (digits--)
*p++ = temp[digits];
if (i < 3)
*p++ = '.';
index += step;
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_compressed_string(char *p, const char *addr)
{
int i, j, range;
unsigned char zerolength[8];
int longest = 1;
int colonpos = -1;
u16 word;
u8 hi, lo;
bool needcolon = false;
bool useIPv4;
struct in6_addr in6;
memcpy(&in6, addr, sizeof(struct in6_addr));
useIPv4 = ipv6_addr_v4mapped(&in6) || ipv6_addr_is_isatap(&in6);
memset(zerolength, 0, sizeof(zerolength));
if (useIPv4)
range = 6;
else
range = 8;
/* find position of longest 0 run */
for (i = 0; i < range; i++) {
for (j = i; j < range; j++) {
if (in6.s6_addr16[j] != 0)
break;
zerolength[i]++;
}
}
for (i = 0; i < range; i++) {
if (zerolength[i] > longest) {
longest = zerolength[i];
colonpos = i;
}
}
if (longest == 1) /* don't compress a single 0 */
colonpos = -1;
/* emit address */
for (i = 0; i < range; i++) {
if (i == colonpos) {
if (needcolon || i == 0)
*p++ = ':';
*p++ = ':';
needcolon = false;
i += longest - 1;
continue;
}
if (needcolon) {
*p++ = ':';
needcolon = false;
}
/* hex u16 without leading 0s */
word = ntohs(in6.s6_addr16[i]);
hi = word >> 8;
lo = word & 0xff;
if (hi) {
if (hi > 0x0f)
p = hex_byte_pack(p, hi);
else
*p++ = hex_asc_lo(hi);
p = hex_byte_pack(p, lo);
}
else if (lo > 0x0f)
p = hex_byte_pack(p, lo);
else
*p++ = hex_asc_lo(lo);
needcolon = true;
}
if (useIPv4) {
if (needcolon)
*p++ = ':';
p = ip4_string(p, &in6.s6_addr[12], "I4");
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_string(char *p, const char *addr, const char *fmt)
{
int i;
for (i = 0; i < 8; i++) {
p = hex_byte_pack(p, *addr++);
p = hex_byte_pack(p, *addr++);
if (fmt[0] == 'I' && i != 7)
*p++ = ':';
}
*p = '\0';
return p;
}
static noinline_for_stack
char *ip6_addr_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char ip6_addr[sizeof("xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255")];
if (fmt[0] == 'I' && fmt[2] == 'c')
ip6_compressed_string(ip6_addr, addr);
else
ip6_string(ip6_addr, addr, fmt);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, ip6_addr, spec);
}
static noinline_for_stack
char *ip4_addr_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char ip4_addr[sizeof("255.255.255.255")];
ip4_string(ip4_addr, addr, fmt);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, ip4_addr, spec);
}
lib: vsprintf: add IPv4/v6 generic %p[Ii]S[pfs] format specifier In order to avoid making code that deals with printing both, IPv4 and IPv6 addresses, unnecessary complicated as for example ... if (sa.sa_family == AF_INET6) printk("... %pI6 ...", ..sin6_addr); else printk("... %pI4 ...", ..sin_addr.s_addr); ... it would be better to introduce a format specifier that can deal with those kind of situations internally; just as we have a "struct sockaddr" for generic mapping into "struct sockaddr_in" or "struct sockaddr_in6" as e.g. done in "union sctp_addr". Then, we could reduce the above statement into something like: printk("... %pIS ..", &sockaddr); In case our pointer is NULL, pointer() then deals with that already at an earlier point in time internally. While we're at it, support for both %piS/%pIS, where 'S' stands for sockaddr, comes (almost) for free. Additionally to that, postfix specifiers 'p', 'f' and 's' are supported as suggested and initially implemented in 2009 by Joe Perches [1]. Handling of those additional specifiers orientate on the initial RFC that was proposed. Also we support IPv6 compressed format specified by 'c' and various other IPv4 extensions as stated in the documentation part. Likely, there are many other areas than just SCTP in the kernel to make use of this extension as well. [1] http://patchwork.ozlabs.org/patch/31480/ Signed-off-by: Daniel Borkmann <dborkman@redhat.com> CC: Joe Perches <joe@perches.com> CC: linux-kernel@vger.kernel.org Signed-off-by: David S. Miller <davem@davemloft.net>
2013-06-29 01:49:39 +08:00
static noinline_for_stack
char *ip6_addr_string_sa(char *buf, char *end, const struct sockaddr_in6 *sa,
struct printf_spec spec, const char *fmt)
{
bool have_p = false, have_s = false, have_f = false, have_c = false;
char ip6_addr[sizeof("[xxxx:xxxx:xxxx:xxxx:xxxx:xxxx:255.255.255.255]") +
sizeof(":12345") + sizeof("/123456789") +
sizeof("%1234567890")];
char *p = ip6_addr, *pend = ip6_addr + sizeof(ip6_addr);
const u8 *addr = (const u8 *) &sa->sin6_addr;
char fmt6[2] = { fmt[0], '6' };
u8 off = 0;
fmt++;
while (isalpha(*++fmt)) {
switch (*fmt) {
case 'p':
have_p = true;
break;
case 'f':
have_f = true;
break;
case 's':
have_s = true;
break;
case 'c':
have_c = true;
break;
}
}
if (have_p || have_s || have_f) {
*p = '[';
off = 1;
}
if (fmt6[0] == 'I' && have_c)
p = ip6_compressed_string(ip6_addr + off, addr);
else
p = ip6_string(ip6_addr + off, addr, fmt6);
if (have_p || have_s || have_f)
*p++ = ']';
if (have_p) {
*p++ = ':';
p = number(p, pend, ntohs(sa->sin6_port), spec);
}
if (have_f) {
*p++ = '/';
p = number(p, pend, ntohl(sa->sin6_flowinfo &
IPV6_FLOWINFO_MASK), spec);
}
if (have_s) {
*p++ = '%';
p = number(p, pend, sa->sin6_scope_id, spec);
}
*p = '\0';
return string(buf, end, ip6_addr, spec);
}
static noinline_for_stack
char *ip4_addr_string_sa(char *buf, char *end, const struct sockaddr_in *sa,
struct printf_spec spec, const char *fmt)
{
bool have_p = false;
char *p, ip4_addr[sizeof("255.255.255.255") + sizeof(":12345")];
char *pend = ip4_addr + sizeof(ip4_addr);
const u8 *addr = (const u8 *) &sa->sin_addr.s_addr;
char fmt4[3] = { fmt[0], '4', 0 };
fmt++;
while (isalpha(*++fmt)) {
switch (*fmt) {
case 'p':
have_p = true;
break;
case 'h':
case 'l':
case 'n':
case 'b':
fmt4[2] = *fmt;
break;
}
}
p = ip4_string(ip4_addr, addr, fmt4);
if (have_p) {
*p++ = ':';
p = number(p, pend, ntohs(sa->sin_port), spec);
}
*p = '\0';
return string(buf, end, ip4_addr, spec);
}
static noinline_for_stack
char *escaped_string(char *buf, char *end, u8 *addr, struct printf_spec spec,
const char *fmt)
{
bool found = true;
int count = 1;
unsigned int flags = 0;
int len;
if (spec.field_width == 0)
return buf; /* nothing to print */
if (ZERO_OR_NULL_PTR(addr))
return string(buf, end, NULL, spec); /* NULL pointer */
do {
switch (fmt[count++]) {
case 'a':
flags |= ESCAPE_ANY;
break;
case 'c':
flags |= ESCAPE_SPECIAL;
break;
case 'h':
flags |= ESCAPE_HEX;
break;
case 'n':
flags |= ESCAPE_NULL;
break;
case 'o':
flags |= ESCAPE_OCTAL;
break;
case 'p':
flags |= ESCAPE_NP;
break;
case 's':
flags |= ESCAPE_SPACE;
break;
default:
found = false;
break;
}
} while (found);
if (!flags)
flags = ESCAPE_ANY_NP;
len = spec.field_width < 0 ? 1 : spec.field_width;
lib/string_helpers.c: change semantics of string_escape_mem The current semantics of string_escape_mem are inadequate for one of its current users, vsnprintf(). If that is to honour its contract, it must know how much space would be needed for the entire escaped buffer, and string_escape_mem provides no way of obtaining that (short of allocating a large enough buffer (~4 times input string) to let it play with, and that's definitely a big no-no inside vsnprintf). So change the semantics for string_escape_mem to be more snprintf-like: Return the size of the output that would be generated if the destination buffer was big enough, but of course still only write to the part of dst it is allowed to, and (contrary to snprintf) don't do '\0'-termination. It is then up to the caller to detect whether output was truncated and to append a '\0' if desired. Also, we must output partial escape sequences, otherwise a call such as snprintf(buf, 3, "%1pE", "\123") would cause printf to write a \0 to buf[2] but leaving buf[0] and buf[1] with whatever they previously contained. This also fixes a bug in the escaped_string() helper function, which used to unconditionally pass a length of "end-buf" to string_escape_mem(); since the latter doesn't check osz for being insanely large, it would happily write to dst. For example, kasprintf(GFP_KERNEL, "something and then %pE", ...); is an easy way to trigger an oops. In test-string_helpers.c, the -ENOMEM test is replaced with testing for getting the expected return value even if the buffer is too small. We also ensure that nothing is written (by relying on a NULL pointer deref) if the output size is 0 by passing NULL - this has to work for kasprintf("%pE") to work. In net/sunrpc/cache.c, I think qword_add still has the same semantics. Someone should definitely double-check this. In fs/proc/array.c, I made the minimum possible change, but longer-term it should stop poking around in seq_file internals. [andriy.shevchenko@linux.intel.com: simplify qword_add] [andriy.shevchenko@linux.intel.com: add missed curly braces] Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Acked-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Signed-off-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-04-16 07:17:28 +08:00
/*
* string_escape_mem() writes as many characters as it can to
* the given buffer, and returns the total size of the output
* had the buffer been big enough.
*/
buf += string_escape_mem(addr, len, buf, buf < end ? end - buf : 0, flags, NULL);
return buf;
}
static noinline_for_stack
char *uuid_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec, const char *fmt)
{
char uuid[sizeof("xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx")];
char *p = uuid;
int i;
static const u8 be[16] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15};
static const u8 le[16] = {3,2,1,0,5,4,7,6,8,9,10,11,12,13,14,15};
const u8 *index = be;
bool uc = false;
switch (*(++fmt)) {
case 'L':
uc = true; /* fall-through */
case 'l':
index = le;
break;
case 'B':
uc = true;
break;
}
for (i = 0; i < 16; i++) {
p = hex_byte_pack(p, addr[index[i]]);
switch (i) {
case 3:
case 5:
case 7:
case 9:
*p++ = '-';
break;
}
}
*p = 0;
if (uc) {
p = uuid;
do {
*p = toupper(*p);
} while (*(++p));
}
return string(buf, end, uuid, spec);
}
static
char *netdev_feature_string(char *buf, char *end, const u8 *addr,
struct printf_spec spec)
{
spec.flags |= SPECIAL | SMALL | ZEROPAD;
if (spec.field_width == -1)
spec.field_width = 2 + 2 * sizeof(netdev_features_t);
spec.base = 16;
return number(buf, end, *(const netdev_features_t *)addr, spec);
}
static noinline_for_stack
char *address_val(char *buf, char *end, const void *addr,
struct printf_spec spec, const char *fmt)
{
unsigned long long num;
spec.flags |= SPECIAL | SMALL | ZEROPAD;
spec.base = 16;
switch (fmt[1]) {
case 'd':
num = *(const dma_addr_t *)addr;
spec.field_width = sizeof(dma_addr_t) * 2 + 2;
break;
case 'p':
default:
num = *(const phys_addr_t *)addr;
spec.field_width = sizeof(phys_addr_t) * 2 + 2;
break;
}
return number(buf, end, num, spec);
}
static noinline_for_stack
char *clock(char *buf, char *end, struct clk *clk, struct printf_spec spec,
const char *fmt)
{
if (!IS_ENABLED(CONFIG_HAVE_CLK) || !clk)
return string(buf, end, NULL, spec);
switch (fmt[1]) {
case 'r':
return number(buf, end, clk_get_rate(clk), spec);
case 'n':
default:
#ifdef CONFIG_COMMON_CLK
return string(buf, end, __clk_get_name(clk), spec);
#else
spec.base = 16;
spec.field_width = sizeof(unsigned long) * 2 + 2;
spec.flags |= SPECIAL | SMALL | ZEROPAD;
return number(buf, end, (unsigned long)clk, spec);
#endif
}
}
int kptr_restrict __read_mostly;
kptr_restrict for hiding kernel pointers from unprivileged users Add the %pK printk format specifier and the /proc/sys/kernel/kptr_restrict sysctl. The %pK format specifier is designed to hide exposed kernel pointers, specifically via /proc interfaces. Exposing these pointers provides an easy target for kernel write vulnerabilities, since they reveal the locations of writable structures containing easily triggerable function pointers. The behavior of %pK depends on the kptr_restrict sysctl. If kptr_restrict is set to 0, no deviation from the standard %p behavior occurs. If kptr_restrict is set to 1, the default, if the current user (intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG (currently in the LSM tree), kernel pointers using %pK are printed as 0's. If kptr_restrict is set to 2, kernel pointers using %pK are printed as 0's regardless of privileges. Replacing with 0's was chosen over the default "(null)", which cannot be parsed by userland %p, which expects "(nil)". [akpm@linux-foundation.org: check for IRQ context when !kptr_restrict, save an indent level, s/WARN/WARN_ONCE/] [akpm@linux-foundation.org: coding-style fixup] [randy.dunlap@oracle.com: fix kernel/sysctl.c warning] Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com> Signed-off-by: Randy Dunlap <randy.dunlap@oracle.com> Cc: James Morris <jmorris@namei.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Thomas Graf <tgraf@infradead.org> Cc: Eugene Teo <eugeneteo@kernel.org> Cc: Kees Cook <kees.cook@canonical.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: David S. Miller <davem@davemloft.net> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Eric Paris <eparis@parisplace.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 08:59:41 +08:00
/*
* Show a '%p' thing. A kernel extension is that the '%p' is followed
* by an extra set of alphanumeric characters that are extended format
* specifiers.
*
* Right now we handle:
*
vsprintf: introduce %pf format specifier A printf format specifier which would allow us to print a pure function name has been suggested by Andrew Morton a couple of months ago. The current %pF is very convenient to print a function symbol, but often we only want to print the name of the function, without its asm offset. That's what %pf does in this patch. The lowecase f has been chosen for its intuitive meaning of a 'weak kind of %pF'. The support for this new format would be welcome by the tracing code where the need to print pure function names is often needed. This is also true for other parts of the kernel: $ git-grep -E "kallsyms_lookup\(.+?\)" arch/blackfin/kernel/traps.c: symname = kallsyms_lookup(address, &symsize, &offset, &modname, namebuf); arch/powerpc/xmon/xmon.c: name = kallsyms_lookup(pc, &size, &offset, NULL, tmpstr); arch/sh/kernel/cpu/sh5/unwind.c: sym = kallsyms_lookup(pc, NULL, &offset, NULL, namebuf); arch/x86/kernel/ftrace.c: kallsyms_lookup((unsigned long) syscall, NULL, NULL, NULL, str); kernel/kprobes.c: sym = kallsyms_lookup((unsigned long)p->addr, NULL, kernel/lockdep.c: return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); kernel/trace/ftrace.c: kallsyms_lookup(*ptr, NULL, NULL, NULL, str); kernel/trace/trace_functions.c: kallsyms_lookup(ip, NULL, NULL, NULL, str); kernel/trace/trace_output.c: kallsyms_lookup(address, NULL, NULL, NULL, str); Changes in v2: - Add the explanation of the %pf role for vsnprintf() and bstr_printf() - Change the comments by dropping the "asm offset" notion and only define the %pf against the actual function offset notion. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Acked-by: Mike Frysinger <vapier@gentoo.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Zhaolei <zhaolei@cn.fujitsu.com> Cc: Tom Zanussi <tzanussi@gmail.com> Cc: Li Zefan <lizf@cn.fujitsu.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <20090415154817.GC5989@nowhere> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-04-15 23:48:18 +08:00
* - 'F' For symbolic function descriptor pointers with offset
* - 'f' For simple symbolic function names without offset
* - 'S' For symbolic direct pointers with offset
* - 's' For symbolic direct pointers without offset
* - '[FfSs]R' as above with __builtin_extract_return_addr() translation
* - 'B' For backtraced symbolic direct pointers with offset
* - 'R' For decoded struct resource, e.g., [mem 0x0-0x1f 64bit pref]
* - 'r' For raw struct resource, e.g., [mem 0x0-0x1f flags 0x201]
lib/vsprintf: implement bitmap printing through '%*pb[l]' bitmap and its derivatives such as cpumask and nodemask currently only provide formatting functions which put the output string into the provided buffer; however, how long this buffer should be isn't defined anywhere and given that some of these bitmaps can be too large to be formatted into an on-stack buffer it users sometimes are unnecessarily forced to come up with creative solutions and compromises for the buffer just to printk these bitmaps. There have been a couple different attempts at making this easier. 1. Way back, PeterZ tried printk '%pb' extension with the precision for bit width - '%.*pb'. This was intuitive and made sense but unfortunately triggered a compile warning about using precision for a pointer. http://lkml.kernel.org/g/1336577562.2527.58.camel@twins 2. I implemented bitmap_pr_cont[_list]() and its wrappers for cpumask and nodemask. This works but PeterZ pointed out that pr_cont's tendency to produce broken lines when multiple CPUs are printing is bothering considering the usages. http://lkml.kernel.org/g/1418226774-30215-3-git-send-email-tj@kernel.org So, this patch is another attempt at teaching printk and friends how to print bitmaps. It's almost identical to what PeterZ tried with precision but it uses the field width for the number of bits instead of precision. The format used is '%*pb[l]', with the optional trailing 'l' specifying list format instead of hex masks. This is a valid format string and doesn't trigger compiler warnings; however, it does make it impossible to specify output field width when printing bitmaps. I think this is an acceptable trade-off given how much easier it makes printing bitmaps and that we don't have any in-kernel user which is using the field width specification. If any future user wants to use field width with a bitmap, it'd have to format the bitmap into a string buffer and then print that buffer with width spec, which isn't different from how it should be done now. This patch implements bitmap[_list]_string() which are called from the vsprintf pointer() formatting function. The implementation is mostly identical to bitmap_scn[list]printf() except that the output is performed in the vsprintf way. These functions handle formatting into too small buffers and sprintf() family of functions report the correct overrun output length. bitmap_scn[list]printf() are now thin wrappers around scnprintf(). Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com> Cc: "John W. Linville" <linville@tuxdriver.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Chris Metcalf <cmetcalf@tilera.com> Cc: Chris Zankel <chris@zankel.net> Cc: Christoph Lameter <cl@linux.com> Cc: Dmitry Torokhov <dmitry.torokhov@gmail.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Li Zefan <lizefan@huawei.com> Cc: Max Filippov <jcmvbkbc@gmail.com> Cc: Mike Travis <travis@sgi.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Steffen Klassert <steffen.klassert@secunet.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-14 06:36:53 +08:00
* - 'b[l]' For a bitmap, the number of bits is determined by the field
* width which must be explicitly specified either as part of the
* format string '%32b[l]' or through '%*b[l]', [l] selects
* range-list format instead of hex format
* - 'M' For a 6-byte MAC address, it prints the address in the
* usual colon-separated hex notation
* - 'm' For a 6-byte MAC address, it prints the hex address without colons
* - 'MF' For a 6-byte MAC FDDI address, it prints the address
* with a dash-separated hex notation
* - '[mM]R' For a 6-byte MAC address, Reverse order (Bluetooth)
* - 'I' [46] for IPv4/IPv6 addresses printed in the usual way
* IPv4 uses dot-separated decimal without leading 0's (1.2.3.4)
* IPv6 uses colon separated network-order 16 bit hex with leading 0's
lib: vsprintf: add IPv4/v6 generic %p[Ii]S[pfs] format specifier In order to avoid making code that deals with printing both, IPv4 and IPv6 addresses, unnecessary complicated as for example ... if (sa.sa_family == AF_INET6) printk("... %pI6 ...", ..sin6_addr); else printk("... %pI4 ...", ..sin_addr.s_addr); ... it would be better to introduce a format specifier that can deal with those kind of situations internally; just as we have a "struct sockaddr" for generic mapping into "struct sockaddr_in" or "struct sockaddr_in6" as e.g. done in "union sctp_addr". Then, we could reduce the above statement into something like: printk("... %pIS ..", &sockaddr); In case our pointer is NULL, pointer() then deals with that already at an earlier point in time internally. While we're at it, support for both %piS/%pIS, where 'S' stands for sockaddr, comes (almost) for free. Additionally to that, postfix specifiers 'p', 'f' and 's' are supported as suggested and initially implemented in 2009 by Joe Perches [1]. Handling of those additional specifiers orientate on the initial RFC that was proposed. Also we support IPv6 compressed format specified by 'c' and various other IPv4 extensions as stated in the documentation part. Likely, there are many other areas than just SCTP in the kernel to make use of this extension as well. [1] http://patchwork.ozlabs.org/patch/31480/ Signed-off-by: Daniel Borkmann <dborkman@redhat.com> CC: Joe Perches <joe@perches.com> CC: linux-kernel@vger.kernel.org Signed-off-by: David S. Miller <davem@davemloft.net>
2013-06-29 01:49:39 +08:00
* [S][pfs]
* Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
* [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
* - 'i' [46] for 'raw' IPv4/IPv6 addresses
* IPv6 omits the colons (01020304...0f)
* IPv4 uses dot-separated decimal with leading 0's (010.123.045.006)
lib: vsprintf: add IPv4/v6 generic %p[Ii]S[pfs] format specifier In order to avoid making code that deals with printing both, IPv4 and IPv6 addresses, unnecessary complicated as for example ... if (sa.sa_family == AF_INET6) printk("... %pI6 ...", ..sin6_addr); else printk("... %pI4 ...", ..sin_addr.s_addr); ... it would be better to introduce a format specifier that can deal with those kind of situations internally; just as we have a "struct sockaddr" for generic mapping into "struct sockaddr_in" or "struct sockaddr_in6" as e.g. done in "union sctp_addr". Then, we could reduce the above statement into something like: printk("... %pIS ..", &sockaddr); In case our pointer is NULL, pointer() then deals with that already at an earlier point in time internally. While we're at it, support for both %piS/%pIS, where 'S' stands for sockaddr, comes (almost) for free. Additionally to that, postfix specifiers 'p', 'f' and 's' are supported as suggested and initially implemented in 2009 by Joe Perches [1]. Handling of those additional specifiers orientate on the initial RFC that was proposed. Also we support IPv6 compressed format specified by 'c' and various other IPv4 extensions as stated in the documentation part. Likely, there are many other areas than just SCTP in the kernel to make use of this extension as well. [1] http://patchwork.ozlabs.org/patch/31480/ Signed-off-by: Daniel Borkmann <dborkman@redhat.com> CC: Joe Perches <joe@perches.com> CC: linux-kernel@vger.kernel.org Signed-off-by: David S. Miller <davem@davemloft.net>
2013-06-29 01:49:39 +08:00
* [S][pfs]
* Generic IPv4/IPv6 address (struct sockaddr *) that falls back to
* [4] or [6] and is able to print port [p], flowinfo [f], scope [s]
* - '[Ii][4S][hnbl]' IPv4 addresses in host, network, big or little endian order
* - 'I[6S]c' for IPv6 addresses printed as specified by
* http://tools.ietf.org/html/rfc5952
* - 'E[achnops]' For an escaped buffer, where rules are defined by combination
* of the following flags (see string_escape_mem() for the
* details):
* a - ESCAPE_ANY
* c - ESCAPE_SPECIAL
* h - ESCAPE_HEX
* n - ESCAPE_NULL
* o - ESCAPE_OCTAL
* p - ESCAPE_NP
* s - ESCAPE_SPACE
* By default ESCAPE_ANY_NP is used.
* - 'U' For a 16 byte UUID/GUID, it prints the UUID/GUID in the form
* "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
* Options for %pU are:
* b big endian lower case hex (default)
* B big endian UPPER case hex
* l little endian lower case hex
* L little endian UPPER case hex
* big endian output byte order is:
* [0][1][2][3]-[4][5]-[6][7]-[8][9]-[10][11][12][13][14][15]
* little endian output byte order is:
* [3][2][1][0]-[5][4]-[7][6]-[8][9]-[10][11][12][13][14][15]
* - 'V' For a struct va_format which contains a format string * and va_list *,
* call vsnprintf(->format, *->va_list).
* Implements a "recursive vsnprintf".
* Do not use this feature without some mechanism to verify the
* correctness of the format string and va_list arguments.
kptr_restrict for hiding kernel pointers from unprivileged users Add the %pK printk format specifier and the /proc/sys/kernel/kptr_restrict sysctl. The %pK format specifier is designed to hide exposed kernel pointers, specifically via /proc interfaces. Exposing these pointers provides an easy target for kernel write vulnerabilities, since they reveal the locations of writable structures containing easily triggerable function pointers. The behavior of %pK depends on the kptr_restrict sysctl. If kptr_restrict is set to 0, no deviation from the standard %p behavior occurs. If kptr_restrict is set to 1, the default, if the current user (intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG (currently in the LSM tree), kernel pointers using %pK are printed as 0's. If kptr_restrict is set to 2, kernel pointers using %pK are printed as 0's regardless of privileges. Replacing with 0's was chosen over the default "(null)", which cannot be parsed by userland %p, which expects "(nil)". [akpm@linux-foundation.org: check for IRQ context when !kptr_restrict, save an indent level, s/WARN/WARN_ONCE/] [akpm@linux-foundation.org: coding-style fixup] [randy.dunlap@oracle.com: fix kernel/sysctl.c warning] Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com> Signed-off-by: Randy Dunlap <randy.dunlap@oracle.com> Cc: James Morris <jmorris@namei.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Thomas Graf <tgraf@infradead.org> Cc: Eugene Teo <eugeneteo@kernel.org> Cc: Kees Cook <kees.cook@canonical.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: David S. Miller <davem@davemloft.net> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Eric Paris <eparis@parisplace.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 08:59:41 +08:00
* - 'K' For a kernel pointer that should be hidden from unprivileged users
* - 'NF' For a netdev_features_t
* - 'h[CDN]' For a variable-length buffer, it prints it as a hex string with
* a certain separator (' ' by default):
* C colon
* D dash
* N no separator
* The maximum supported length is 64 bytes of the input. Consider
* to use print_hex_dump() for the larger input.
* - 'a[pd]' For address types [p] phys_addr_t, [d] dma_addr_t and derivatives
* (default assumed to be phys_addr_t, passed by reference)
* - 'd[234]' For a dentry name (optionally 2-4 last components)
* - 'D[234]' Same as 'd' but for a struct file
* - 'g' For block_device name (gendisk + partition number)
* - 'C' For a clock, it prints the name (Common Clock Framework) or address
* (legacy clock framework) of the clock
* - 'Cn' For a clock, it prints the name (Common Clock Framework) or address
* (legacy clock framework) of the clock
* - 'Cr' For a clock, it prints the current rate of the clock
*
* ** Please update also Documentation/printk-formats.txt when making changes **
*
* Note: The difference between 'S' and 'F' is that on ia64 and ppc64
* function pointers are really function descriptors, which contain a
* pointer to the real address.
*/
static noinline_for_stack
char *pointer(const char *fmt, char *buf, char *end, void *ptr,
struct printf_spec spec)
{
const int default_width = 2 * sizeof(void *);
if (!ptr && *fmt != 'K') {
/*
* Print (null) with the same width as a pointer so it makes
* tabular output look nice.
*/
if (spec.field_width == -1)
spec.field_width = default_width;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return string(buf, end, "(null)", spec);
}
switch (*fmt) {
case 'F':
vsprintf: introduce %pf format specifier A printf format specifier which would allow us to print a pure function name has been suggested by Andrew Morton a couple of months ago. The current %pF is very convenient to print a function symbol, but often we only want to print the name of the function, without its asm offset. That's what %pf does in this patch. The lowecase f has been chosen for its intuitive meaning of a 'weak kind of %pF'. The support for this new format would be welcome by the tracing code where the need to print pure function names is often needed. This is also true for other parts of the kernel: $ git-grep -E "kallsyms_lookup\(.+?\)" arch/blackfin/kernel/traps.c: symname = kallsyms_lookup(address, &symsize, &offset, &modname, namebuf); arch/powerpc/xmon/xmon.c: name = kallsyms_lookup(pc, &size, &offset, NULL, tmpstr); arch/sh/kernel/cpu/sh5/unwind.c: sym = kallsyms_lookup(pc, NULL, &offset, NULL, namebuf); arch/x86/kernel/ftrace.c: kallsyms_lookup((unsigned long) syscall, NULL, NULL, NULL, str); kernel/kprobes.c: sym = kallsyms_lookup((unsigned long)p->addr, NULL, kernel/lockdep.c: return kallsyms_lookup((unsigned long)key, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, NULL, str); kernel/trace/ftrace.c: kallsyms_lookup(rec->ip, NULL, NULL, &modname, str); kernel/trace/ftrace.c: kallsyms_lookup(*ptr, NULL, NULL, NULL, str); kernel/trace/trace_functions.c: kallsyms_lookup(ip, NULL, NULL, NULL, str); kernel/trace/trace_output.c: kallsyms_lookup(address, NULL, NULL, NULL, str); Changes in v2: - Add the explanation of the %pf role for vsnprintf() and bstr_printf() - Change the comments by dropping the "asm offset" notion and only define the %pf against the actual function offset notion. Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Acked-by: Mike Frysinger <vapier@gentoo.org> Cc: Linus Torvalds <torvalds@linux-foundation.org> Cc: Zhaolei <zhaolei@cn.fujitsu.com> Cc: Tom Zanussi <tzanussi@gmail.com> Cc: Li Zefan <lizf@cn.fujitsu.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <20090415154817.GC5989@nowhere> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-04-15 23:48:18 +08:00
case 'f':
ptr = dereference_function_descriptor(ptr);
/* Fallthrough */
case 'S':
case 's':
case 'B':
return symbol_string(buf, end, ptr, spec, fmt);
case 'R':
case 'r':
return resource_string(buf, end, ptr, spec, fmt);
case 'h':
return hex_string(buf, end, ptr, spec, fmt);
lib/vsprintf: implement bitmap printing through '%*pb[l]' bitmap and its derivatives such as cpumask and nodemask currently only provide formatting functions which put the output string into the provided buffer; however, how long this buffer should be isn't defined anywhere and given that some of these bitmaps can be too large to be formatted into an on-stack buffer it users sometimes are unnecessarily forced to come up with creative solutions and compromises for the buffer just to printk these bitmaps. There have been a couple different attempts at making this easier. 1. Way back, PeterZ tried printk '%pb' extension with the precision for bit width - '%.*pb'. This was intuitive and made sense but unfortunately triggered a compile warning about using precision for a pointer. http://lkml.kernel.org/g/1336577562.2527.58.camel@twins 2. I implemented bitmap_pr_cont[_list]() and its wrappers for cpumask and nodemask. This works but PeterZ pointed out that pr_cont's tendency to produce broken lines when multiple CPUs are printing is bothering considering the usages. http://lkml.kernel.org/g/1418226774-30215-3-git-send-email-tj@kernel.org So, this patch is another attempt at teaching printk and friends how to print bitmaps. It's almost identical to what PeterZ tried with precision but it uses the field width for the number of bits instead of precision. The format used is '%*pb[l]', with the optional trailing 'l' specifying list format instead of hex masks. This is a valid format string and doesn't trigger compiler warnings; however, it does make it impossible to specify output field width when printing bitmaps. I think this is an acceptable trade-off given how much easier it makes printing bitmaps and that we don't have any in-kernel user which is using the field width specification. If any future user wants to use field width with a bitmap, it'd have to format the bitmap into a string buffer and then print that buffer with width spec, which isn't different from how it should be done now. This patch implements bitmap[_list]_string() which are called from the vsprintf pointer() formatting function. The implementation is mostly identical to bitmap_scn[list]printf() except that the output is performed in the vsprintf way. These functions handle formatting into too small buffers and sprintf() family of functions report the correct overrun output length. bitmap_scn[list]printf() are now thin wrappers around scnprintf(). Signed-off-by: Tejun Heo <tj@kernel.org> Acked-by: Peter Zijlstra (Intel) <peterz@infradead.org> Cc: "David S. Miller" <davem@davemloft.net> Cc: "James E.J. Bottomley" <James.Bottomley@HansenPartnership.com> Cc: "John W. Linville" <linville@tuxdriver.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Benjamin Herrenschmidt <benh@kernel.crashing.org> Cc: Chris Metcalf <cmetcalf@tilera.com> Cc: Chris Zankel <chris@zankel.net> Cc: Christoph Lameter <cl@linux.com> Cc: Dmitry Torokhov <dmitry.torokhov@gmail.com> Cc: Fenghua Yu <fenghua.yu@intel.com> Cc: Greg Kroah-Hartman <gregkh@linuxfoundation.org> Cc: Ingo Molnar <mingo@redhat.com> Cc: Li Zefan <lizefan@huawei.com> Cc: Max Filippov <jcmvbkbc@gmail.com> Cc: Mike Travis <travis@sgi.com> Cc: Pekka Enberg <penberg@kernel.org> Cc: Russell King <linux@arm.linux.org.uk> Cc: Rusty Russell <rusty@rustcorp.com.au> Cc: Steffen Klassert <steffen.klassert@secunet.com> Cc: Steven Rostedt <rostedt@goodmis.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Tony Luck <tony.luck@intel.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-02-14 06:36:53 +08:00
case 'b':
switch (fmt[1]) {
case 'l':
return bitmap_list_string(buf, end, ptr, spec, fmt);
default:
return bitmap_string(buf, end, ptr, spec, fmt);
}
case 'M': /* Colon separated: 00:01:02:03:04:05 */
case 'm': /* Contiguous: 000102030405 */
/* [mM]F (FDDI) */
/* [mM]R (Reverse order; Bluetooth) */
return mac_address_string(buf, end, ptr, spec, fmt);
case 'I': /* Formatted IP supported
* 4: 1.2.3.4
* 6: 0001:0203:...:0708
* 6c: 1::708 or 1::1.2.3.4
*/
case 'i': /* Contiguous:
* 4: 001.002.003.004
* 6: 000102...0f
*/
switch (fmt[1]) {
case '6':
return ip6_addr_string(buf, end, ptr, spec, fmt);
case '4':
return ip4_addr_string(buf, end, ptr, spec, fmt);
lib: vsprintf: add IPv4/v6 generic %p[Ii]S[pfs] format specifier In order to avoid making code that deals with printing both, IPv4 and IPv6 addresses, unnecessary complicated as for example ... if (sa.sa_family == AF_INET6) printk("... %pI6 ...", ..sin6_addr); else printk("... %pI4 ...", ..sin_addr.s_addr); ... it would be better to introduce a format specifier that can deal with those kind of situations internally; just as we have a "struct sockaddr" for generic mapping into "struct sockaddr_in" or "struct sockaddr_in6" as e.g. done in "union sctp_addr". Then, we could reduce the above statement into something like: printk("... %pIS ..", &sockaddr); In case our pointer is NULL, pointer() then deals with that already at an earlier point in time internally. While we're at it, support for both %piS/%pIS, where 'S' stands for sockaddr, comes (almost) for free. Additionally to that, postfix specifiers 'p', 'f' and 's' are supported as suggested and initially implemented in 2009 by Joe Perches [1]. Handling of those additional specifiers orientate on the initial RFC that was proposed. Also we support IPv6 compressed format specified by 'c' and various other IPv4 extensions as stated in the documentation part. Likely, there are many other areas than just SCTP in the kernel to make use of this extension as well. [1] http://patchwork.ozlabs.org/patch/31480/ Signed-off-by: Daniel Borkmann <dborkman@redhat.com> CC: Joe Perches <joe@perches.com> CC: linux-kernel@vger.kernel.org Signed-off-by: David S. Miller <davem@davemloft.net>
2013-06-29 01:49:39 +08:00
case 'S': {
const union {
struct sockaddr raw;
struct sockaddr_in v4;
struct sockaddr_in6 v6;
} *sa = ptr;
switch (sa->raw.sa_family) {
case AF_INET:
return ip4_addr_string_sa(buf, end, &sa->v4, spec, fmt);
case AF_INET6:
return ip6_addr_string_sa(buf, end, &sa->v6, spec, fmt);
default:
return string(buf, end, "(invalid address)", spec);
}}
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case 'E':
return escaped_string(buf, end, ptr, spec, fmt);
case 'U':
return uuid_string(buf, end, ptr, spec, fmt);
case 'V':
{
va_list va;
va_copy(va, *((struct va_format *)ptr)->va);
buf += vsnprintf(buf, end > buf ? end - buf : 0,
((struct va_format *)ptr)->fmt, va);
va_end(va);
return buf;
}
kptr_restrict for hiding kernel pointers from unprivileged users Add the %pK printk format specifier and the /proc/sys/kernel/kptr_restrict sysctl. The %pK format specifier is designed to hide exposed kernel pointers, specifically via /proc interfaces. Exposing these pointers provides an easy target for kernel write vulnerabilities, since they reveal the locations of writable structures containing easily triggerable function pointers. The behavior of %pK depends on the kptr_restrict sysctl. If kptr_restrict is set to 0, no deviation from the standard %p behavior occurs. If kptr_restrict is set to 1, the default, if the current user (intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG (currently in the LSM tree), kernel pointers using %pK are printed as 0's. If kptr_restrict is set to 2, kernel pointers using %pK are printed as 0's regardless of privileges. Replacing with 0's was chosen over the default "(null)", which cannot be parsed by userland %p, which expects "(nil)". [akpm@linux-foundation.org: check for IRQ context when !kptr_restrict, save an indent level, s/WARN/WARN_ONCE/] [akpm@linux-foundation.org: coding-style fixup] [randy.dunlap@oracle.com: fix kernel/sysctl.c warning] Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com> Signed-off-by: Randy Dunlap <randy.dunlap@oracle.com> Cc: James Morris <jmorris@namei.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Thomas Graf <tgraf@infradead.org> Cc: Eugene Teo <eugeneteo@kernel.org> Cc: Kees Cook <kees.cook@canonical.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: David S. Miller <davem@davemloft.net> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Eric Paris <eparis@parisplace.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 08:59:41 +08:00
case 'K':
/*
* %pK cannot be used in IRQ context because its test
* for CAP_SYSLOG would be meaningless.
*/
if (kptr_restrict && (in_irq() || in_serving_softirq() ||
in_nmi())) {
kptr_restrict for hiding kernel pointers from unprivileged users Add the %pK printk format specifier and the /proc/sys/kernel/kptr_restrict sysctl. The %pK format specifier is designed to hide exposed kernel pointers, specifically via /proc interfaces. Exposing these pointers provides an easy target for kernel write vulnerabilities, since they reveal the locations of writable structures containing easily triggerable function pointers. The behavior of %pK depends on the kptr_restrict sysctl. If kptr_restrict is set to 0, no deviation from the standard %p behavior occurs. If kptr_restrict is set to 1, the default, if the current user (intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG (currently in the LSM tree), kernel pointers using %pK are printed as 0's. If kptr_restrict is set to 2, kernel pointers using %pK are printed as 0's regardless of privileges. Replacing with 0's was chosen over the default "(null)", which cannot be parsed by userland %p, which expects "(nil)". [akpm@linux-foundation.org: check for IRQ context when !kptr_restrict, save an indent level, s/WARN/WARN_ONCE/] [akpm@linux-foundation.org: coding-style fixup] [randy.dunlap@oracle.com: fix kernel/sysctl.c warning] Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com> Signed-off-by: Randy Dunlap <randy.dunlap@oracle.com> Cc: James Morris <jmorris@namei.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Thomas Graf <tgraf@infradead.org> Cc: Eugene Teo <eugeneteo@kernel.org> Cc: Kees Cook <kees.cook@canonical.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: David S. Miller <davem@davemloft.net> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Eric Paris <eparis@parisplace.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 08:59:41 +08:00
if (spec.field_width == -1)
spec.field_width = default_width;
kptr_restrict for hiding kernel pointers from unprivileged users Add the %pK printk format specifier and the /proc/sys/kernel/kptr_restrict sysctl. The %pK format specifier is designed to hide exposed kernel pointers, specifically via /proc interfaces. Exposing these pointers provides an easy target for kernel write vulnerabilities, since they reveal the locations of writable structures containing easily triggerable function pointers. The behavior of %pK depends on the kptr_restrict sysctl. If kptr_restrict is set to 0, no deviation from the standard %p behavior occurs. If kptr_restrict is set to 1, the default, if the current user (intended to be a reader via seq_printf(), etc.) does not have CAP_SYSLOG (currently in the LSM tree), kernel pointers using %pK are printed as 0's. If kptr_restrict is set to 2, kernel pointers using %pK are printed as 0's regardless of privileges. Replacing with 0's was chosen over the default "(null)", which cannot be parsed by userland %p, which expects "(nil)". [akpm@linux-foundation.org: check for IRQ context when !kptr_restrict, save an indent level, s/WARN/WARN_ONCE/] [akpm@linux-foundation.org: coding-style fixup] [randy.dunlap@oracle.com: fix kernel/sysctl.c warning] Signed-off-by: Dan Rosenberg <drosenberg@vsecurity.com> Signed-off-by: Randy Dunlap <randy.dunlap@oracle.com> Cc: James Morris <jmorris@namei.org> Cc: Eric Dumazet <eric.dumazet@gmail.com> Cc: Thomas Graf <tgraf@infradead.org> Cc: Eugene Teo <eugeneteo@kernel.org> Cc: Kees Cook <kees.cook@canonical.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: David S. Miller <davem@davemloft.net> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Eric Paris <eparis@parisplace.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2011-01-13 08:59:41 +08:00
return string(buf, end, "pK-error", spec);
}
vsprintf: check real user/group id for %pK Some setuid binaries will allow reading of files which have read permission by the real user id. This is problematic with files which use %pK because the file access permission is checked at open() time, but the kptr_restrict setting is checked at read() time. If a setuid binary opens a %pK file as an unprivileged user, and then elevates permissions before reading the file, then kernel pointer values may be leaked. This happens for example with the setuid pppd application on Ubuntu 12.04: $ head -1 /proc/kallsyms 00000000 T startup_32 $ pppd file /proc/kallsyms pppd: In file /proc/kallsyms: unrecognized option 'c1000000' This will only leak the pointer value from the first line, but other setuid binaries may leak more information. Fix this by adding a check that in addition to the current process having CAP_SYSLOG, that effective user and group ids are equal to the real ids. If a setuid binary reads the contents of a file which uses %pK then the pointer values will be printed as NULL if the real user is unprivileged. Update the sysctl documentation to reflect the changes, and also correct the documentation to state the kptr_restrict=0 is the default. This is a only temporary solution to the issue. The correct solution is to do the permission check at open() time on files, and to replace %pK with a function which checks the open() time permission. %pK uses in printk should be removed since no sane permission check can be done, and instead protected by using dmesg_restrict. Signed-off-by: Ryan Mallon <rmallon@gmail.com> Cc: Kees Cook <keescook@chromium.org> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Joe Perches <joe@perches.com> Cc: "Eric W. Biederman" <ebiederm@xmission.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-13 07:08:51 +08:00
switch (kptr_restrict) {
case 0:
/* Always print %pK values */
break;
case 1: {
/*
* Only print the real pointer value if the current
* process has CAP_SYSLOG and is running with the
* same credentials it started with. This is because
* access to files is checked at open() time, but %pK
* checks permission at read() time. We don't want to
* leak pointer values if a binary opens a file using
* %pK and then elevates privileges before reading it.
*/
const struct cred *cred = current_cred();
if (!has_capability_noaudit(current, CAP_SYSLOG) ||
!uid_eq(cred->euid, cred->uid) ||
!gid_eq(cred->egid, cred->gid))
ptr = NULL;
break;
}
case 2:
default:
/* Always print 0's for %pK */
ptr = NULL;
vsprintf: check real user/group id for %pK Some setuid binaries will allow reading of files which have read permission by the real user id. This is problematic with files which use %pK because the file access permission is checked at open() time, but the kptr_restrict setting is checked at read() time. If a setuid binary opens a %pK file as an unprivileged user, and then elevates permissions before reading the file, then kernel pointer values may be leaked. This happens for example with the setuid pppd application on Ubuntu 12.04: $ head -1 /proc/kallsyms 00000000 T startup_32 $ pppd file /proc/kallsyms pppd: In file /proc/kallsyms: unrecognized option 'c1000000' This will only leak the pointer value from the first line, but other setuid binaries may leak more information. Fix this by adding a check that in addition to the current process having CAP_SYSLOG, that effective user and group ids are equal to the real ids. If a setuid binary reads the contents of a file which uses %pK then the pointer values will be printed as NULL if the real user is unprivileged. Update the sysctl documentation to reflect the changes, and also correct the documentation to state the kptr_restrict=0 is the default. This is a only temporary solution to the issue. The correct solution is to do the permission check at open() time on files, and to replace %pK with a function which checks the open() time permission. %pK uses in printk should be removed since no sane permission check can be done, and instead protected by using dmesg_restrict. Signed-off-by: Ryan Mallon <rmallon@gmail.com> Cc: Kees Cook <keescook@chromium.org> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Joe Perches <joe@perches.com> Cc: "Eric W. Biederman" <ebiederm@xmission.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-13 07:08:51 +08:00
break;
}
break;
vsprintf: check real user/group id for %pK Some setuid binaries will allow reading of files which have read permission by the real user id. This is problematic with files which use %pK because the file access permission is checked at open() time, but the kptr_restrict setting is checked at read() time. If a setuid binary opens a %pK file as an unprivileged user, and then elevates permissions before reading the file, then kernel pointer values may be leaked. This happens for example with the setuid pppd application on Ubuntu 12.04: $ head -1 /proc/kallsyms 00000000 T startup_32 $ pppd file /proc/kallsyms pppd: In file /proc/kallsyms: unrecognized option 'c1000000' This will only leak the pointer value from the first line, but other setuid binaries may leak more information. Fix this by adding a check that in addition to the current process having CAP_SYSLOG, that effective user and group ids are equal to the real ids. If a setuid binary reads the contents of a file which uses %pK then the pointer values will be printed as NULL if the real user is unprivileged. Update the sysctl documentation to reflect the changes, and also correct the documentation to state the kptr_restrict=0 is the default. This is a only temporary solution to the issue. The correct solution is to do the permission check at open() time on files, and to replace %pK with a function which checks the open() time permission. %pK uses in printk should be removed since no sane permission check can be done, and instead protected by using dmesg_restrict. Signed-off-by: Ryan Mallon <rmallon@gmail.com> Cc: Kees Cook <keescook@chromium.org> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Joe Perches <joe@perches.com> Cc: "Eric W. Biederman" <ebiederm@xmission.com> Cc: <stable@vger.kernel.org> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2013-11-13 07:08:51 +08:00
case 'N':
switch (fmt[1]) {
case 'F':
return netdev_feature_string(buf, end, ptr, spec);
}
break;
case 'a':
return address_val(buf, end, ptr, spec, fmt);
case 'd':
return dentry_name(buf, end, ptr, spec, fmt);
case 'C':
return clock(buf, end, ptr, spec, fmt);
case 'D':
return dentry_name(buf, end,
((const struct file *)ptr)->f_path.dentry,
spec, fmt);
#ifdef CONFIG_BLOCK
case 'g':
return bdev_name(buf, end, ptr, spec, fmt);
#endif
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
spec.flags |= SMALL;
if (spec.field_width == -1) {
spec.field_width = default_width;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.flags |= ZEROPAD;
}
spec.base = 16;
return number(buf, end, (unsigned long) ptr, spec);
}
/*
* Helper function to decode printf style format.
* Each call decode a token from the format and return the
* number of characters read (or likely the delta where it wants
* to go on the next call).
* The decoded token is returned through the parameters
*
* 'h', 'l', or 'L' for integer fields
* 'z' support added 23/7/1999 S.H.
* 'z' changed to 'Z' --davidm 1/25/99
* 't' added for ptrdiff_t
*
* @fmt: the format string
* @type of the token returned
* @flags: various flags such as +, -, # tokens..
* @field_width: overwritten width
* @base: base of the number (octal, hex, ...)
* @precision: precision of a number
* @qualifier: qualifier of a number (long, size_t, ...)
*/
static noinline_for_stack
int format_decode(const char *fmt, struct printf_spec *spec)
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
{
const char *start = fmt;
/* we finished early by reading the field width */
if (spec->type == FORMAT_TYPE_WIDTH) {
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (spec->field_width < 0) {
spec->field_width = -spec->field_width;
spec->flags |= LEFT;
}
spec->type = FORMAT_TYPE_NONE;
goto precision;
}
/* we finished early by reading the precision */
if (spec->type == FORMAT_TYPE_PRECISION) {
if (spec->precision < 0)
spec->precision = 0;
spec->type = FORMAT_TYPE_NONE;
goto qualifier;
}
/* By default */
spec->type = FORMAT_TYPE_NONE;
for (; *fmt ; ++fmt) {
if (*fmt == '%')
break;
}
/* Return the current non-format string */
if (fmt != start || !*fmt)
return fmt - start;
/* Process flags */
spec->flags = 0;
while (1) { /* this also skips first '%' */
bool found = true;
++fmt;
switch (*fmt) {
case '-': spec->flags |= LEFT; break;
case '+': spec->flags |= PLUS; break;
case ' ': spec->flags |= SPACE; break;
case '#': spec->flags |= SPECIAL; break;
case '0': spec->flags |= ZEROPAD; break;
default: found = false;
}
if (!found)
break;
}
/* get field width */
spec->field_width = -1;
if (isdigit(*fmt))
spec->field_width = skip_atoi(&fmt);
else if (*fmt == '*') {
/* it's the next argument */
spec->type = FORMAT_TYPE_WIDTH;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return ++fmt - start;
}
precision:
/* get the precision */
spec->precision = -1;
if (*fmt == '.') {
++fmt;
if (isdigit(*fmt)) {
spec->precision = skip_atoi(&fmt);
if (spec->precision < 0)
spec->precision = 0;
} else if (*fmt == '*') {
/* it's the next argument */
spec->type = FORMAT_TYPE_PRECISION;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return ++fmt - start;
}
}
qualifier:
/* get the conversion qualifier */
spec->qualifier = -1;
if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
_tolower(*fmt) == 'z' || *fmt == 't') {
spec->qualifier = *fmt++;
if (unlikely(spec->qualifier == *fmt)) {
if (spec->qualifier == 'l') {
spec->qualifier = 'L';
++fmt;
} else if (spec->qualifier == 'h') {
spec->qualifier = 'H';
++fmt;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
}
/* default base */
spec->base = 10;
switch (*fmt) {
case 'c':
spec->type = FORMAT_TYPE_CHAR;
return ++fmt - start;
case 's':
spec->type = FORMAT_TYPE_STR;
return ++fmt - start;
case 'p':
spec->type = FORMAT_TYPE_PTR;
return ++fmt - start;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case '%':
spec->type = FORMAT_TYPE_PERCENT_CHAR;
return ++fmt - start;
/* integer number formats - set up the flags and "break" */
case 'o':
spec->base = 8;
break;
case 'x':
spec->flags |= SMALL;
case 'X':
spec->base = 16;
break;
case 'd':
case 'i':
spec->flags |= SIGN;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case 'u':
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case 'n':
/*
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
* Since %n poses a greater security risk than
* utility, treat it as any other invalid or
* unsupported format specifier.
*/
/* Fall-through */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
default:
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
WARN_ONCE(1, "Please remove unsupported %%%c in format string\n", *fmt);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec->type = FORMAT_TYPE_INVALID;
return fmt - start;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (spec->qualifier == 'L')
spec->type = FORMAT_TYPE_LONG_LONG;
else if (spec->qualifier == 'l') {
BUILD_BUG_ON(FORMAT_TYPE_ULONG + SIGN != FORMAT_TYPE_LONG);
spec->type = FORMAT_TYPE_ULONG + (spec->flags & SIGN);
} else if (_tolower(spec->qualifier) == 'z') {
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec->type = FORMAT_TYPE_SIZE_T;
} else if (spec->qualifier == 't') {
spec->type = FORMAT_TYPE_PTRDIFF;
} else if (spec->qualifier == 'H') {
BUILD_BUG_ON(FORMAT_TYPE_UBYTE + SIGN != FORMAT_TYPE_BYTE);
spec->type = FORMAT_TYPE_UBYTE + (spec->flags & SIGN);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
} else if (spec->qualifier == 'h') {
BUILD_BUG_ON(FORMAT_TYPE_USHORT + SIGN != FORMAT_TYPE_SHORT);
spec->type = FORMAT_TYPE_USHORT + (spec->flags & SIGN);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
} else {
BUILD_BUG_ON(FORMAT_TYPE_UINT + SIGN != FORMAT_TYPE_INT);
spec->type = FORMAT_TYPE_UINT + (spec->flags & SIGN);
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
return ++fmt - start;
}
/**
* vsnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @args: Arguments for the format string
*
* This function generally follows C99 vsnprintf, but has some
* extensions and a few limitations:
*
* %n is unsupported
* %p* is handled by pointer()
*
* See pointer() or Documentation/printk-formats.txt for more
* extensive description.
*
* ** Please update the documentation in both places when making changes **
*
* The return value is the number of characters which would
* be generated for the given input, excluding the trailing
* '\0', as per ISO C99. If you want to have the exact
* number of characters written into @buf as return value
* (not including the trailing '\0'), use vscnprintf(). If the
* return is greater than or equal to @size, the resulting
* string is truncated.
*
* If you're not already dealing with a va_list consider using snprintf().
*/
int vsnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
unsigned long long num;
char *str, *end;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
struct printf_spec spec = {0};
/* Reject out-of-range values early. Large positive sizes are
used for unknown buffer sizes. */
if (WARN_ON_ONCE(size > INT_MAX))
return 0;
str = buf;
end = buf + size;
/* Make sure end is always >= buf */
if (end < buf) {
end = ((void *)-1);
size = end - buf;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (*fmt) {
const char *old_fmt = fmt;
int read = format_decode(fmt, &spec);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
fmt += read;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
switch (spec.type) {
case FORMAT_TYPE_NONE: {
int copy = read;
if (str < end) {
if (copy > end - str)
copy = end - str;
memcpy(str, old_fmt, copy);
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
str += read;
break;
}
case FORMAT_TYPE_WIDTH:
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width = va_arg(args, int);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PRECISION:
spec.precision = va_arg(args, int);
break;
case FORMAT_TYPE_CHAR: {
char c;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (!(spec.flags & LEFT)) {
while (--spec.field_width > 0) {
if (str < end)
*str = ' ';
++str;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
}
c = (unsigned char) va_arg(args, int);
if (str < end)
*str = c;
++str;
while (--spec.field_width > 0) {
if (str < end)
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
*str = ' ';
++str;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
break;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_STR:
str = string(str, end, va_arg(args, char *), spec);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PTR:
str = pointer(fmt, str, end, va_arg(args, void *),
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec);
while (isalnum(*fmt))
fmt++;
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PERCENT_CHAR:
if (str < end)
*str = '%';
++str;
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_INVALID:
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
/*
* Presumably the arguments passed gcc's type
* checking, but there is no safe or sane way
* for us to continue parsing the format and
* fetching from the va_list; the remaining
* specifiers and arguments would be out of
* sync.
*/
goto out;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
default:
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
num = va_arg(args, long long);
break;
case FORMAT_TYPE_ULONG:
num = va_arg(args, unsigned long);
break;
case FORMAT_TYPE_LONG:
num = va_arg(args, long);
break;
case FORMAT_TYPE_SIZE_T:
if (spec.flags & SIGN)
num = va_arg(args, ssize_t);
else
num = va_arg(args, size_t);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_PTRDIFF:
num = va_arg(args, ptrdiff_t);
break;
case FORMAT_TYPE_UBYTE:
num = (unsigned char) va_arg(args, int);
break;
case FORMAT_TYPE_BYTE:
num = (signed char) va_arg(args, int);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_USHORT:
num = (unsigned short) va_arg(args, int);
break;
case FORMAT_TYPE_SHORT:
num = (short) va_arg(args, int);
break;
case FORMAT_TYPE_INT:
num = (int) va_arg(args, int);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
default:
num = va_arg(args, unsigned int);
}
str = number(str, end, num, spec);
}
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
out:
if (size > 0) {
if (str < end)
*str = '\0';
else
end[-1] = '\0';
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
/* the trailing null byte doesn't count towards the total */
return str-buf;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
EXPORT_SYMBOL(vsnprintf);
/**
* vscnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The return value is the number of characters which have been written into
* the @buf not including the trailing '\0'. If @size is == 0 the function
* returns 0.
*
* If you're not already dealing with a va_list consider using scnprintf().
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int vscnprintf(char *buf, size_t size, const char *fmt, va_list args)
{
int i;
i = vsnprintf(buf, size, fmt, args);
if (likely(i < size))
return i;
if (size != 0)
return size - 1;
return 0;
}
EXPORT_SYMBOL(vscnprintf);
/**
* snprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The return value is the number of characters which would be
* generated for the given input, excluding the trailing null,
* as per ISO C99. If the return is greater than or equal to
* @size, the resulting string is truncated.
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int snprintf(char *buf, size_t size, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsnprintf(buf, size, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(snprintf);
/**
* scnprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The return value is the number of characters written into @buf not including
* the trailing '\0'. If @size is == 0 the function returns 0.
*/
int scnprintf(char *buf, size_t size, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vscnprintf(buf, size, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(scnprintf);
/**
* vsprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The function returns the number of characters written
* into @buf. Use vsnprintf() or vscnprintf() in order to avoid
* buffer overflows.
*
* If you're not already dealing with a va_list consider using sprintf().
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int vsprintf(char *buf, const char *fmt, va_list args)
{
return vsnprintf(buf, INT_MAX, fmt, args);
}
EXPORT_SYMBOL(vsprintf);
/**
* sprintf - Format a string and place it in a buffer
* @buf: The buffer to place the result into
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The function returns the number of characters written
* into @buf. Use snprintf() or scnprintf() in order to avoid
* buffer overflows.
*
* See the vsnprintf() documentation for format string extensions over C99.
*/
int sprintf(char *buf, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsnprintf(buf, INT_MAX, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(sprintf);
#ifdef CONFIG_BINARY_PRINTF
/*
* bprintf service:
* vbin_printf() - VA arguments to binary data
* bstr_printf() - Binary data to text string
*/
/**
* vbin_printf - Parse a format string and place args' binary value in a buffer
* @bin_buf: The buffer to place args' binary value
* @size: The size of the buffer(by words(32bits), not characters)
* @fmt: The format string to use
* @args: Arguments for the format string
*
* The format follows C99 vsnprintf, except %n is ignored, and its argument
* is skipped.
*
* The return value is the number of words(32bits) which would be generated for
* the given input.
*
* NOTE:
* If the return value is greater than @size, the resulting bin_buf is NOT
* valid for bstr_printf().
*/
int vbin_printf(u32 *bin_buf, size_t size, const char *fmt, va_list args)
{
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
struct printf_spec spec = {0};
char *str, *end;
str = (char *)bin_buf;
end = (char *)(bin_buf + size);
#define save_arg(type) \
do { \
if (sizeof(type) == 8) { \
unsigned long long value; \
str = PTR_ALIGN(str, sizeof(u32)); \
value = va_arg(args, unsigned long long); \
if (str + sizeof(type) <= end) { \
*(u32 *)str = *(u32 *)&value; \
*(u32 *)(str + 4) = *((u32 *)&value + 1); \
} \
} else { \
unsigned long value; \
str = PTR_ALIGN(str, sizeof(type)); \
value = va_arg(args, int); \
if (str + sizeof(type) <= end) \
*(typeof(type) *)str = (type)value; \
} \
str += sizeof(type); \
} while (0)
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (*fmt) {
int read = format_decode(fmt, &spec);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
fmt += read;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
switch (spec.type) {
case FORMAT_TYPE_NONE:
case FORMAT_TYPE_PERCENT_CHAR:
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
case FORMAT_TYPE_INVALID:
goto out;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_WIDTH:
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PRECISION:
save_arg(int);
break;
case FORMAT_TYPE_CHAR:
save_arg(char);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_STR: {
const char *save_str = va_arg(args, char *);
size_t len;
if ((unsigned long)save_str > (unsigned long)-PAGE_SIZE
|| (unsigned long)save_str < PAGE_SIZE)
save_str = "(null)";
len = strlen(save_str) + 1;
if (str + len < end)
memcpy(str, save_str, len);
str += len;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PTR:
save_arg(void *);
/* skip all alphanumeric pointer suffixes */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (isalnum(*fmt))
fmt++;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
default:
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
save_arg(long long);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_ULONG:
case FORMAT_TYPE_LONG:
save_arg(unsigned long);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_SIZE_T:
save_arg(size_t);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_PTRDIFF:
save_arg(ptrdiff_t);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
case FORMAT_TYPE_UBYTE:
case FORMAT_TYPE_BYTE:
save_arg(char);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_USHORT:
case FORMAT_TYPE_SHORT:
save_arg(short);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
default:
save_arg(int);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
}
}
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
out:
return (u32 *)(PTR_ALIGN(str, sizeof(u32))) - bin_buf;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
#undef save_arg
}
EXPORT_SYMBOL_GPL(vbin_printf);
/**
* bstr_printf - Format a string from binary arguments and place it in a buffer
* @buf: The buffer to place the result into
* @size: The size of the buffer, including the trailing null space
* @fmt: The format string to use
* @bin_buf: Binary arguments for the format string
*
* This function like C99 vsnprintf, but the difference is that vsnprintf gets
* arguments from stack, and bstr_printf gets arguments from @bin_buf which is
* a binary buffer that generated by vbin_printf.
*
* The format follows C99 vsnprintf, but has some extensions:
* see vsnprintf comment for details.
*
* The return value is the number of characters which would
* be generated for the given input, excluding the trailing
* '\0', as per ISO C99. If you want to have the exact
* number of characters written into @buf as return value
* (not including the trailing '\0'), use vscnprintf(). If the
* return is greater than or equal to @size, the resulting
* string is truncated.
*/
int bstr_printf(char *buf, size_t size, const char *fmt, const u32 *bin_buf)
{
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
struct printf_spec spec = {0};
char *str, *end;
const char *args = (const char *)bin_buf;
if (WARN_ON_ONCE(size > INT_MAX))
return 0;
str = buf;
end = buf + size;
#define get_arg(type) \
({ \
typeof(type) value; \
if (sizeof(type) == 8) { \
args = PTR_ALIGN(args, sizeof(u32)); \
*(u32 *)&value = *(u32 *)args; \
*((u32 *)&value + 1) = *(u32 *)(args + 4); \
} else { \
args = PTR_ALIGN(args, sizeof(type)); \
value = *(typeof(type) *)args; \
} \
args += sizeof(type); \
value; \
})
/* Make sure end is always >= buf */
if (end < buf) {
end = ((void *)-1);
size = end - buf;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (*fmt) {
const char *old_fmt = fmt;
int read = format_decode(fmt, &spec);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
fmt += read;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
switch (spec.type) {
case FORMAT_TYPE_NONE: {
int copy = read;
if (str < end) {
if (copy > end - str)
copy = end - str;
memcpy(str, old_fmt, copy);
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
str += read;
break;
}
case FORMAT_TYPE_WIDTH:
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
spec.field_width = get_arg(int);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PRECISION:
spec.precision = get_arg(int);
break;
case FORMAT_TYPE_CHAR: {
char c;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
if (!(spec.flags & LEFT)) {
while (--spec.field_width > 0) {
if (str < end)
*str = ' ';
++str;
}
}
c = (unsigned char) get_arg(char);
if (str < end)
*str = c;
++str;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (--spec.field_width > 0) {
if (str < end)
*str = ' ';
++str;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_STR: {
const char *str_arg = args;
args += strlen(str_arg) + 1;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
str = string(str, end, (char *)str_arg, spec);
break;
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PTR:
str = pointer(fmt, str, end, get_arg(void *), spec);
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
while (isalnum(*fmt))
fmt++;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_PERCENT_CHAR:
if (str < end)
*str = '%';
++str;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
break;
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
case FORMAT_TYPE_INVALID:
goto out;
default: {
unsigned long long num;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
switch (spec.type) {
case FORMAT_TYPE_LONG_LONG:
num = get_arg(long long);
break;
case FORMAT_TYPE_ULONG:
case FORMAT_TYPE_LONG:
num = get_arg(unsigned long);
break;
case FORMAT_TYPE_SIZE_T:
num = get_arg(size_t);
break;
case FORMAT_TYPE_PTRDIFF:
num = get_arg(ptrdiff_t);
break;
case FORMAT_TYPE_UBYTE:
num = get_arg(unsigned char);
break;
case FORMAT_TYPE_BYTE:
num = get_arg(signed char);
break;
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
case FORMAT_TYPE_USHORT:
num = get_arg(unsigned short);
break;
case FORMAT_TYPE_SHORT:
num = get_arg(short);
break;
case FORMAT_TYPE_UINT:
num = get_arg(unsigned int);
break;
default:
num = get_arg(int);
}
str = number(str, end, num, spec);
} /* default: */
} /* switch(spec.type) */
} /* while(*fmt) */
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
lib/vsprintf.c: handle invalid format specifiers more robustly If we meet any invalid or unsupported format specifier, 'handling' it by just printing it as a literal string is not safe: Presumably the format string and the arguments passed gcc's type checking, but that means something like sprintf(buf, "%n %pd", &intvar, dentry) would end up interpreting &intvar as a struct dentry*. When the offending specifier was %n it used to be at the end of the format string, but we can't rely on that always being the case. Also, gcc doesn't complain about some more or less exotic qualifiers (or 'length modifiers' in posix-speak) such as 'j' or 'q', but being unrecognized by the kernel's printf implementation, they'd be interpreted as unknown specifiers, and the rest of arguments would be interpreted wrongly. So let's complain about anything we don't understand, not just %n, and stop pretending that we'd be able to make sense of the rest of the format/arguments. If the offending specifier is in a printk() call we unfortunately only get a "BUG: recent printk recursion!", but at least direct users of the sprintf family will be caught. Signed-off-by: Rasmus Villemoes <linux@rasmusvillemoes.dk> Reviewed-by: Andy Shevchenko <andriy.shevchenko@linux.intel.com> Acked-by: Kees Cook <keescook@chromium.org> Cc: Martin Kletzander <mkletzan@redhat.com> Cc: Rasmus Villemoes <linux@rasmusvillemoes.dk> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2015-11-07 08:30:20 +08:00
out:
if (size > 0) {
if (str < end)
*str = '\0';
else
end[-1] = '\0';
}
vsprintf: unify the format decoding layer for its 3 users An new optimization is making its way to ftrace. Its purpose is to make trace_printk() consuming less memory and be faster. Written by Lai Jiangshan, the approach is to delay the formatting job from tracing time to output time. Currently, a call to trace_printk() will format the whole string and insert it into the ring buffer. Then you can read it on /debug/tracing/trace file. The new implementation stores the address of the format string and the binary parameters into the ring buffer, making the packet more compact and faster to insert. Later, when the user exports the traces, the format string is retrieved with the binary parameters and the formatting job is eventually done. The new implementation rewrites a lot of format decoding bits from vsnprintf() function, making now 3 differents functions to maintain in their duplicated parts of printf format decoding bits. Suggested by Ingo Molnar, this patch tries to factorize the most possible common bits from these functions. The real common part between them is the format decoding. Although they do somewhat similar jobs, their way to export or import the parameters is very different. Thus, only the decoding layer is extracted, unless you see other parts that could be worth factorized. Changes in V2: - Address a suggestion from Linus to group the format_decode() parameters inside a structure. Changes in v3: - Address other cleanups suggested by Ingo and Linus such as passing the printf_spec struct to the format helpers: pointer()/number()/string() Note that this struct is passed by copy and not by address. This is to avoid side effects because these functions often change these values and the changes shoudn't be persistant when a callee helper returns. It would be too risky. - Various cleanups (code alignement, switch/case instead of if/else fountains). - Fix a bug that printed the first format specifier following a %p Changes in v4: - drop unapropriate const qualifier loss while casting fmt to a char * (thanks to Vegard Nossum for having pointed this out). Signed-off-by: Frederic Weisbecker <fweisbec@gmail.com> Cc: Linus Torvalds <torvalds@linux-foundation.org> Acked-by: Steven Rostedt <rostedt@goodmis.org> LKML-Reference: <1236356510-8381-6-git-send-email-fweisbec@gmail.com> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-03-07 00:21:50 +08:00
#undef get_arg
/* the trailing null byte doesn't count towards the total */
return str - buf;
}
EXPORT_SYMBOL_GPL(bstr_printf);
/**
* bprintf - Parse a format string and place args' binary value in a buffer
* @bin_buf: The buffer to place args' binary value
* @size: The size of the buffer(by words(32bits), not characters)
* @fmt: The format string to use
* @...: Arguments for the format string
*
* The function returns the number of words(u32) written
* into @bin_buf.
*/
int bprintf(u32 *bin_buf, size_t size, const char *fmt, ...)
{
va_list args;
int ret;
va_start(args, fmt);
ret = vbin_printf(bin_buf, size, fmt, args);
va_end(args);
return ret;
}
EXPORT_SYMBOL_GPL(bprintf);
#endif /* CONFIG_BINARY_PRINTF */
/**
* vsscanf - Unformat a buffer into a list of arguments
* @buf: input buffer
* @fmt: format of buffer
* @args: arguments
*/
int vsscanf(const char *buf, const char *fmt, va_list args)
{
const char *str = buf;
char *next;
char digit;
int num = 0;
u8 qualifier;
unsigned int base;
union {
long long s;
unsigned long long u;
} val;
s16 field_width;
bool is_sign;
while (*fmt) {
/* skip any white space in format */
/* white space in format matchs any amount of
* white space, including none, in the input.
*/
if (isspace(*fmt)) {
tree-wide: convert open calls to remove spaces to skip_spaces() lib function Makes use of skip_spaces() defined in lib/string.c for removing leading spaces from strings all over the tree. It decreases lib.a code size by 47 bytes and reuses the function tree-wide: text data bss dec hex filename 64688 584 592 65864 10148 (TOTALS-BEFORE) 64641 584 592 65817 10119 (TOTALS-AFTER) Also, while at it, if we see (*str && isspace(*str)), we can be sure to remove the first condition (*str) as the second one (isspace(*str)) also evaluates to 0 whenever *str == 0, making it redundant. In other words, "a char equals zero is never a space". Julia Lawall tried the semantic patch (http://coccinelle.lip6.fr) below, and found occurrences of this pattern on 3 more files: drivers/leds/led-class.c drivers/leds/ledtrig-timer.c drivers/video/output.c @@ expression str; @@ ( // ignore skip_spaces cases while (*str && isspace(*str)) { \(str++;\|++str;\) } | - *str && isspace(*str) ) Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com> Cc: Julia Lawall <julia@diku.dk> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Jeff Dike <jdike@addtoit.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Richard Purdie <rpurdie@rpsys.net> Cc: Neil Brown <neilb@suse.de> Cc: Kyle McMartin <kyle@mcmartin.ca> Cc: Henrique de Moraes Holschuh <hmh@hmh.eng.br> Cc: David Howells <dhowells@redhat.com> Cc: <linux-ext4@vger.kernel.org> Cc: Samuel Ortiz <samuel@sortiz.org> Cc: Patrick McHardy <kaber@trash.net> Cc: Takashi Iwai <tiwai@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 10:01:06 +08:00
fmt = skip_spaces(++fmt);
str = skip_spaces(str);
}
/* anything that is not a conversion must match exactly */
if (*fmt != '%' && *fmt) {
if (*fmt++ != *str++)
break;
continue;
}
if (!*fmt)
break;
++fmt;
/* skip this conversion.
* advance both strings to next white space
*/
if (*fmt == '*') {
if (!*str)
break;
while (!isspace(*fmt) && *fmt != '%' && *fmt)
fmt++;
while (!isspace(*str) && *str)
str++;
continue;
}
/* get field width */
field_width = -1;
if (isdigit(*fmt)) {
field_width = skip_atoi(&fmt);
if (field_width <= 0)
break;
}
/* get conversion qualifier */
qualifier = -1;
if (*fmt == 'h' || _tolower(*fmt) == 'l' ||
_tolower(*fmt) == 'z') {
qualifier = *fmt++;
if (unlikely(qualifier == *fmt)) {
if (qualifier == 'h') {
qualifier = 'H';
fmt++;
} else if (qualifier == 'l') {
qualifier = 'L';
fmt++;
}
}
}
if (!*fmt)
break;
if (*fmt == 'n') {
/* return number of characters read so far */
*va_arg(args, int *) = str - buf;
++fmt;
continue;
}
if (!*str)
break;
base = 10;
is_sign = false;
switch (*fmt++) {
case 'c':
{
char *s = (char *)va_arg(args, char*);
if (field_width == -1)
field_width = 1;
do {
*s++ = *str++;
} while (--field_width > 0 && *str);
num++;
}
continue;
case 's':
{
char *s = (char *)va_arg(args, char *);
if (field_width == -1)
field_width = SHRT_MAX;
/* first, skip leading white space in buffer */
tree-wide: convert open calls to remove spaces to skip_spaces() lib function Makes use of skip_spaces() defined in lib/string.c for removing leading spaces from strings all over the tree. It decreases lib.a code size by 47 bytes and reuses the function tree-wide: text data bss dec hex filename 64688 584 592 65864 10148 (TOTALS-BEFORE) 64641 584 592 65817 10119 (TOTALS-AFTER) Also, while at it, if we see (*str && isspace(*str)), we can be sure to remove the first condition (*str) as the second one (isspace(*str)) also evaluates to 0 whenever *str == 0, making it redundant. In other words, "a char equals zero is never a space". Julia Lawall tried the semantic patch (http://coccinelle.lip6.fr) below, and found occurrences of this pattern on 3 more files: drivers/leds/led-class.c drivers/leds/ledtrig-timer.c drivers/video/output.c @@ expression str; @@ ( // ignore skip_spaces cases while (*str && isspace(*str)) { \(str++;\|++str;\) } | - *str && isspace(*str) ) Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com> Cc: Julia Lawall <julia@diku.dk> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Jeff Dike <jdike@addtoit.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Richard Purdie <rpurdie@rpsys.net> Cc: Neil Brown <neilb@suse.de> Cc: Kyle McMartin <kyle@mcmartin.ca> Cc: Henrique de Moraes Holschuh <hmh@hmh.eng.br> Cc: David Howells <dhowells@redhat.com> Cc: <linux-ext4@vger.kernel.org> Cc: Samuel Ortiz <samuel@sortiz.org> Cc: Patrick McHardy <kaber@trash.net> Cc: Takashi Iwai <tiwai@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 10:01:06 +08:00
str = skip_spaces(str);
/* now copy until next white space */
while (*str && !isspace(*str) && field_width--)
*s++ = *str++;
*s = '\0';
num++;
}
continue;
case 'o':
base = 8;
break;
case 'x':
case 'X':
base = 16;
break;
case 'i':
base = 0;
case 'd':
is_sign = true;
case 'u':
break;
case '%':
/* looking for '%' in str */
if (*str++ != '%')
return num;
continue;
default:
/* invalid format; stop here */
return num;
}
/* have some sort of integer conversion.
* first, skip white space in buffer.
*/
tree-wide: convert open calls to remove spaces to skip_spaces() lib function Makes use of skip_spaces() defined in lib/string.c for removing leading spaces from strings all over the tree. It decreases lib.a code size by 47 bytes and reuses the function tree-wide: text data bss dec hex filename 64688 584 592 65864 10148 (TOTALS-BEFORE) 64641 584 592 65817 10119 (TOTALS-AFTER) Also, while at it, if we see (*str && isspace(*str)), we can be sure to remove the first condition (*str) as the second one (isspace(*str)) also evaluates to 0 whenever *str == 0, making it redundant. In other words, "a char equals zero is never a space". Julia Lawall tried the semantic patch (http://coccinelle.lip6.fr) below, and found occurrences of this pattern on 3 more files: drivers/leds/led-class.c drivers/leds/ledtrig-timer.c drivers/video/output.c @@ expression str; @@ ( // ignore skip_spaces cases while (*str && isspace(*str)) { \(str++;\|++str;\) } | - *str && isspace(*str) ) Signed-off-by: André Goddard Rosa <andre.goddard@gmail.com> Cc: Julia Lawall <julia@diku.dk> Cc: Martin Schwidefsky <schwidefsky@de.ibm.com> Cc: Jeff Dike <jdike@addtoit.com> Cc: Ingo Molnar <mingo@elte.hu> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: "H. Peter Anvin" <hpa@zytor.com> Cc: Richard Purdie <rpurdie@rpsys.net> Cc: Neil Brown <neilb@suse.de> Cc: Kyle McMartin <kyle@mcmartin.ca> Cc: Henrique de Moraes Holschuh <hmh@hmh.eng.br> Cc: David Howells <dhowells@redhat.com> Cc: <linux-ext4@vger.kernel.org> Cc: Samuel Ortiz <samuel@sortiz.org> Cc: Patrick McHardy <kaber@trash.net> Cc: Takashi Iwai <tiwai@suse.de> Signed-off-by: Andrew Morton <akpm@linux-foundation.org> Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
2009-12-15 10:01:06 +08:00
str = skip_spaces(str);
digit = *str;
if (is_sign && digit == '-')
digit = *(str + 1);
if (!digit
|| (base == 16 && !isxdigit(digit))
|| (base == 10 && !isdigit(digit))
|| (base == 8 && (!isdigit(digit) || digit > '7'))
|| (base == 0 && !isdigit(digit)))
break;
if (is_sign)
val.s = qualifier != 'L' ?
simple_strtol(str, &next, base) :
simple_strtoll(str, &next, base);
else
val.u = qualifier != 'L' ?
simple_strtoul(str, &next, base) :
simple_strtoull(str, &next, base);
if (field_width > 0 && next - str > field_width) {
if (base == 0)
_parse_integer_fixup_radix(str, &base);
while (next - str > field_width) {
if (is_sign)
val.s = div_s64(val.s, base);
else
val.u = div_u64(val.u, base);
--next;
}
}
switch (qualifier) {
case 'H': /* that's 'hh' in format */
if (is_sign)
*va_arg(args, signed char *) = val.s;
else
*va_arg(args, unsigned char *) = val.u;
break;
case 'h':
if (is_sign)
*va_arg(args, short *) = val.s;
else
*va_arg(args, unsigned short *) = val.u;
break;
case 'l':
if (is_sign)
*va_arg(args, long *) = val.s;
else
*va_arg(args, unsigned long *) = val.u;
break;
case 'L':
if (is_sign)
*va_arg(args, long long *) = val.s;
else
*va_arg(args, unsigned long long *) = val.u;
break;
case 'Z':
case 'z':
*va_arg(args, size_t *) = val.u;
break;
default:
if (is_sign)
*va_arg(args, int *) = val.s;
else
*va_arg(args, unsigned int *) = val.u;
break;
}
num++;
if (!next)
break;
str = next;
}
return num;
}
EXPORT_SYMBOL(vsscanf);
/**
* sscanf - Unformat a buffer into a list of arguments
* @buf: input buffer
* @fmt: formatting of buffer
* @...: resulting arguments
*/
int sscanf(const char *buf, const char *fmt, ...)
{
va_list args;
int i;
va_start(args, fmt);
i = vsscanf(buf, fmt, args);
va_end(args);
return i;
}
EXPORT_SYMBOL(sscanf);