maxjhandsome
/
llvm-project
forked from OSchip/llvm-project
1
0
Fork 0
Code Issues Pull Requests Packages Releases Wiki Activity

Allow VarStreamArray to use stateful extractors. 

Previously extractors tried to be stateless with any additional
context information needed in order to parse items being passed
in via the extraction method.  This led to quite cumbersome
implementation challenges and awkwardness of use.  This patch
brings back support for stateful extractors, making the
implementation and usage simpler.

llvm-svn: 305093
This commit is contained in:
Zachary Turner 2017-06-09 17:54:36 +00:00
parent d9de6389fc
commit 7e62cd17d6
15 changed files with 143 additions and 217 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
llvm/include/llvm/DebugInfo/CodeView/CVRecord.h
View File

@ -62,10 +62,8 @@ template <typename Kind> struct RemappedRecord {
template <typename Kind>
struct VarStreamArrayExtractor<codeview::CVRecord<Kind>> {
typedef void ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Len,
codeview::CVRecord<Kind> &Item) {
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
codeview::CVRecord<Kind> &Item) {
using namespace codeview;
const RecordPrefix *Prefix = nullptr;
BinaryStreamReader Reader(Stream);

4
llvm/include/llvm/DebugInfo/CodeView/DebugChecksumsSubsection.h
View File

@ -36,8 +36,8 @@ template <> struct VarStreamArrayExtractor<codeview::FileChecksumEntry> {
public:
typedef void ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Len,
codeview::FileChecksumEntry &Item);
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
codeview::FileChecksumEntry &Item);
};
}

4
llvm/include/llvm/DebugInfo/CodeView/DebugCrossImpSubsection.h
View File

@ -31,8 +31,8 @@ template <> struct VarStreamArrayExtractor<codeview::CrossModuleImportItem> {
public:
typedef void ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Len,
codeview::CrossModuleImportItem &Item);
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
codeview::CrossModuleImportItem &Item);
};
}

7
llvm/include/llvm/DebugInfo/CodeView/DebugInlineeLinesSubsection.h
View File

@ -43,10 +43,9 @@ struct InlineeSourceLine {
}
template <> struct VarStreamArrayExtractor<codeview::InlineeSourceLine> {
typedef bool ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Len,
codeview::InlineeSourceLine &Item, bool HasExtraFiles);
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
codeview::InlineeSourceLine &Item);
bool HasExtraFiles = false;
};
namespace codeview {

6
llvm/include/llvm/DebugInfo/CodeView/DebugLinesSubsection.h
View File

@ -64,10 +64,10 @@ struct LineColumnEntry {
class LineColumnExtractor {
public:
typedef const LineFragmentHeader *ContextType;
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
LineColumnEntry &Item);
static Error extract(BinaryStreamRef Stream, uint32_t &Len,
LineColumnEntry &Item, const LineFragmentHeader *Ctx);
const LineFragmentHeader *Header = nullptr;
};
class DebugLinesSubsectionRef final : public DebugSubsectionRef {

6
llvm/include/llvm/DebugInfo/CodeView/DebugSubsectionRecord.h
View File

@ -62,10 +62,8 @@ private:
} // namespace codeview
template <> struct VarStreamArrayExtractor<codeview::DebugSubsectionRecord> {
typedef void ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Length,
codeview::DebugSubsectionRecord &Info) {
Error operator()(BinaryStreamRef Stream, uint32_t &Length,
codeview::DebugSubsectionRecord &Info) {
if (auto EC = codeview::DebugSubsectionRecord::initialize(
Stream, Info, codeview::CodeViewContainer::Pdb))
return EC;

5
llvm/include/llvm/DebugInfo/PDB/Native/DbiModuleDescriptor.h
View File

@ -56,9 +56,8 @@ private:
} // end namespace pdb
template <> struct VarStreamArrayExtractor<pdb::DbiModuleDescriptor> {
typedef void ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Length,
pdb::DbiModuleDescriptor &Info) {
Error operator()(BinaryStreamRef Stream, uint32_t &Length,
pdb::DbiModuleDescriptor &Info) {
if (auto EC = pdb::DbiModuleDescriptor::initialize(Stream, Info))
return EC;
Length = Info.getRecordLength();

262
llvm/include/llvm/Support/BinaryStreamArray.h
View File

@ -42,36 +42,114 @@ namespace llvm {
/// having to specify a second template argument to VarStreamArray (documented
/// below).
template <typename T> struct VarStreamArrayExtractor {
struct ContextType {};
// Method intentionally deleted. You must provide an explicit specialization
// with one of the following two methods implemented.
static Error extract(BinaryStreamRef Stream, uint32_t &Len, T &Item) = delete;
static Error extract(BinaryStreamRef Stream, uint32_t &Len, T &Item,
const ContextType &Ctx) = delete;
// with the following method implemented.
Error operator()(BinaryStreamRef Stream, uint32_t &Len,
T &Item) const = delete;
};
template <typename ArrayType, typename Value, typename Extractor,
typename WrappedCtx>
class VarStreamArrayIterator
: public iterator_facade_base<
VarStreamArrayIterator<ArrayType, Value, Extractor, WrappedCtx>,
std::forward_iterator_tag, Value> {
typedef VarStreamArrayIterator<ArrayType, Value, Extractor, WrappedCtx>
IterType;
/// VarStreamArray represents an array of variable length records backed by a
/// stream. This could be a contiguous sequence of bytes in memory, it could
/// be a file on disk, or it could be a PDB stream where bytes are stored as
/// discontiguous blocks in a file. Usually it is desirable to treat arrays
/// as contiguous blocks of memory, but doing so with large PDB files, for
/// example, could mean allocating huge amounts of memory just to allow
/// re-ordering of stream data to be contiguous before iterating over it. By
/// abstracting this out, we need not duplicate this memory, and we can
/// iterate over arrays in arbitrarily formatted streams. Elements are parsed
/// lazily on iteration, so there is no upfront cost associated with building
/// or copying a VarStreamArray, no matter how large it may be.
///
/// You create a VarStreamArray by specifying a ValueType and an Extractor type.
/// If you do not specify an Extractor type, you are expected to specialize
/// VarStreamArrayExtractor<T> for your ValueType.
///
/// By default an Extractor is default constructed in the class, but in some
/// cases you might find it useful for an Extractor to maintain state across
/// extractions. In this case you can provide your own Extractor through a
/// secondary constructor. The following examples show various ways of
/// creating a VarStreamArray.
///
/// // Will use VarStreamArrayExtractor<MyType> as the extractor.
/// VarStreamArray<MyType> MyTypeArray;
///
/// // Will use a default-constructed MyExtractor as the extractor.
/// VarStreamArray<MyType, MyExtractor> MyTypeArray2;
///
/// // Will use the specific instance of MyExtractor provided.
/// // MyExtractor need not be default-constructible in this case.
/// MyExtractor E(SomeContext);
/// VarStreamArray<MyType, MyExtractor> MyTypeArray3(E);
///
template <typename ValueType, typename Extractor> class VarStreamArrayIterator;
template <typename ValueType,
typename Extractor = VarStreamArrayExtractor<ValueType>>
class VarStreamArray {
friend class VarStreamArrayIterator<ValueType, Extractor>;
public:
VarStreamArrayIterator() = default;
VarStreamArrayIterator(const ArrayType &Array, const WrappedCtx &Ctx,
BinaryStreamRef Stream, bool *HadError = nullptr,
uint32_t Offset = 0)
: IterRef(Stream), Ctx(&Ctx), Array(&Array), AbsOffset(Offset),
HadError(HadError) {
typedef VarStreamArrayIterator<ValueType, Extractor> Iterator;
VarStreamArray() = default;
explicit VarStreamArray(const Extractor &E) : E(E) {}
explicit VarStreamArray(BinaryStreamRef Stream) : Stream(Stream) {}
VarStreamArray(BinaryStreamRef Stream, const Extractor &E)
: Stream(Stream), E(E) {}
Iterator begin(bool *HadError = nullptr) const {
return Iterator(*this, E, HadError);
}
bool valid() const { return Stream.valid(); }
Iterator end() const { return Iterator(E); }
bool empty() const { return Stream.getLength() == 0; }
/// \brief given an offset into the array's underlying stream, return an
/// iterator to the record at that offset. This is considered unsafe
/// since the behavior is undefined if \p Offset does not refer to the
/// beginning of a valid record.
Iterator at(uint32_t Offset) const {
return Iterator(*this, E, Offset, nullptr);
}
const Extractor &getExtractor() const { return E; }
Extractor &getExtractor() { return E; }
BinaryStreamRef getUnderlyingStream() const { return Stream; }
void setUnderlyingStream(BinaryStreamRef S) { Stream = S; }
private:
BinaryStreamRef Stream;
Extractor E;
};
template <typename ValueType, typename Extractor>
class VarStreamArrayIterator
: public iterator_facade_base<VarStreamArrayIterator<ValueType, Extractor>,
std::forward_iterator_tag, ValueType> {
typedef VarStreamArrayIterator<ValueType, Extractor> IterType;
typedef VarStreamArray<ValueType, Extractor> ArrayType;
public:
VarStreamArrayIterator(const ArrayType &Array, const Extractor &E,
bool *HadError)
: VarStreamArrayIterator(Array, E, 0, HadError) {}
VarStreamArrayIterator(const ArrayType &Array, const Extractor &E,
uint32_t Offset, bool *HadError)
: IterRef(Array.Stream.drop_front(Offset)), Array(&Array),
AbsOffset(Offset), HadError(HadError), Extract(E) {
if (IterRef.getLength() == 0)
moveToEnd();
else {
auto EC = Ctx.template invoke<Extractor>(IterRef, ThisLen, ThisValue);
auto EC = Extract(IterRef, ThisLen, ThisValue);
if (EC) {
consumeError(std::move(EC));
markError();
@ -79,13 +157,8 @@ public:
}
}
VarStreamArrayIterator(const ArrayType &Array, const WrappedCtx &Ctx,
bool *HadError = nullptr)
: VarStreamArrayIterator(Array, Ctx, Array.Stream, HadError) {}
VarStreamArrayIterator(const WrappedCtx &Ctx) : Ctx(&Ctx) {}
VarStreamArrayIterator(const VarStreamArrayIterator &Other) = default;
VarStreamArrayIterator() = default;
explicit VarStreamArrayIterator(const Extractor &E) : Extract(E) {}
~VarStreamArrayIterator() = default;
bool operator==(const IterType &R) const {
@ -103,12 +176,12 @@ public:
return false;
}
const Value &operator*() const {
const ValueType &operator*() const {
assert(Array && !HasError);
return ThisValue;
}
Value &operator*() {
ValueType &operator*() {
assert(Array && !HasError);
return ThisValue;
}
@ -125,7 +198,7 @@ public:
moveToEnd();
} else {
// There is some data after the current record.
auto EC = Ctx->template invoke<Extractor>(IterRef, ThisLen, ThisValue);
auto EC = Extract(IterRef, ThisLen, ThisValue);
if (EC) {
consumeError(std::move(EC));
markError();
@ -153,9 +226,9 @@ private:
*HadError = true;
}
Value ThisValue;
ValueType ThisValue;
BinaryStreamRef IterRef;
const WrappedCtx *Ctx{nullptr};
Extractor Extract;
const ArrayType *Array{nullptr};
uint32_t ThisLen{0};
uint32_t AbsOffset{0};
@ -163,127 +236,6 @@ private:
bool *HadError{nullptr};
};
template <typename T, typename Context> struct ContextWrapper {
ContextWrapper() = default;
explicit ContextWrapper(Context &&Ctx) : Ctx(Ctx) {}
template <typename Extractor>
Error invoke(BinaryStreamRef Stream, uint32_t &Len, T &Item) const {
return Extractor::extract(Stream, Len, Item, Ctx);
}
Context Ctx;
};
template <typename T> struct ContextWrapper<T, void> {
ContextWrapper() = default;
template <typename Extractor>
Error invoke(BinaryStreamRef Stream, uint32_t &Len, T &Item) const {
return Extractor::extract(Stream, Len, Item);
}
};
/// VarStreamArray represents an array of variable length records backed by a
/// stream. This could be a contiguous sequence of bytes in memory, it could
/// be a file on disk, or it could be a PDB stream where bytes are stored as
/// discontiguous blocks in a file. Usually it is desirable to treat arrays
/// as contiguous blocks of memory, but doing so with large PDB files, for
/// example, could mean allocating huge amounts of memory just to allow
/// re-ordering of stream data to be contiguous before iterating over it. By
/// abstracting this out, we need not duplicate this memory, and we can
/// iterate over arrays in arbitrarily formatted streams. Elements are parsed
/// lazily on iteration, so there is no upfront cost associated with building
/// or copying a VarStreamArray, no matter how large it may be.
///
/// You create a VarStreamArray by specifying a ValueType and an Extractor type.
/// If you do not specify an Extractor type, you are expected to specialize
/// VarStreamArrayExtractor<T> for your ValueType.
///
/// The default extractor type is stateless, but by specializing
/// VarStreamArrayExtractor or defining your own custom extractor type and
/// adding the appropriate ContextType typedef to the class, you can pass a
/// context field during construction of the VarStreamArray that will be
/// passed to each call to extract.
///
template <typename Value, typename Extractor, typename WrappedCtx>
class VarStreamArrayBase {
typedef VarStreamArrayBase<Value, Extractor, WrappedCtx> MyType;
public:
typedef VarStreamArrayIterator<MyType, Value, Extractor, WrappedCtx> Iterator;
friend Iterator;
VarStreamArrayBase() = default;
VarStreamArrayBase(BinaryStreamRef Stream, const WrappedCtx &Ctx)
: Stream(Stream), Ctx(Ctx) {}
VarStreamArrayBase(const MyType &Other)
: Stream(Other.Stream), Ctx(Other.Ctx) {}
Iterator begin(bool *HadError = nullptr) const {
if (empty())
return end();
return Iterator(*this, Ctx, Stream, HadError);
}
bool valid() const { return Stream.valid(); }
Iterator end() const { return Iterator(Ctx); }
bool empty() const { return Stream.getLength() == 0; }
/// \brief given an offset into the array's underlying stream, return an
/// iterator to the record at that offset. This is considered unsafe
/// since the behavior is undefined if \p Offset does not refer to the
/// beginning of a valid record.
Iterator at(uint32_t Offset) const {
return Iterator(*this, Ctx, Stream.drop_front(Offset), nullptr, Offset);
}
BinaryStreamRef getUnderlyingStream() const { return Stream; }
private:
BinaryStreamRef Stream;
WrappedCtx Ctx;
};
template <typename Value, typename Extractor, typename Context>
class VarStreamArrayImpl
: public VarStreamArrayBase<Value, Extractor,
ContextWrapper<Value, Context>> {
typedef ContextWrapper<Value, Context> WrappedContext;
typedef VarStreamArrayImpl<Value, Extractor, Context> MyType;
typedef VarStreamArrayBase<Value, Extractor, WrappedContext> BaseType;
public:
typedef Context ContextType;
VarStreamArrayImpl() = default;
VarStreamArrayImpl(BinaryStreamRef Stream, Context &&Ctx)
: BaseType(Stream, WrappedContext(std::forward<Context>(Ctx))) {}
};
template <typename Value, typename Extractor>
class VarStreamArrayImpl<Value, Extractor, void>
: public VarStreamArrayBase<Value, Extractor, ContextWrapper<Value, void>> {
typedef ContextWrapper<Value, void> WrappedContext;
typedef VarStreamArrayImpl<Value, Extractor, void> MyType;
typedef VarStreamArrayBase<Value, Extractor, WrappedContext> BaseType;
public:
VarStreamArrayImpl() = default;
VarStreamArrayImpl(BinaryStreamRef Stream)
: BaseType(Stream, WrappedContext()) {}
};
template <typename Value, typename Extractor = VarStreamArrayExtractor<Value>>
using VarStreamArray =
VarStreamArrayImpl<Value, Extractor, typename Extractor::ContextType>;
template <typename T> class FixedStreamArrayIterator;
/// FixedStreamArray is similar to VarStreamArray, except with each record

20
llvm/include/llvm/Support/BinaryStreamReader.h
View File

@ -198,25 +198,7 @@ public:
BinaryStreamRef S;
if (auto EC = readStreamRef(S, Size))
return EC;
Array = VarStreamArray<T, U>(S);
return Error::success();
}
/// Read a VarStreamArray of size \p Size bytes and store the result into
/// \p Array. Updates the stream's offset to point after the newly read
/// array. Never causes a copy (although iterating the elements of the
/// VarStreamArray may, depending upon the implementation of the underlying
/// stream).
///
/// \returns a success error code if the data was successfully read, otherwise
/// returns an appropriate error code.
template <typename T, typename U, typename ContextType>
Error readArray(VarStreamArray<T, U> &Array, uint32_t Size,
ContextType &&Context) {
BinaryStreamRef S;
if (auto EC = readStreamRef(S, Size))
return EC;
Array = VarStreamArray<T, U>(S, std::move(Context));
Array.setUnderlyingStream(S);
return Error::success();
}

4
llvm/lib/DebugInfo/CodeView/DebugChecksumsSubsection.cpp
View File

@ -25,8 +25,8 @@ struct FileChecksumEntryHeader {
// Checksum bytes follow.
};
Error llvm::VarStreamArrayExtractor<FileChecksumEntry>::extract(
BinaryStreamRef Stream, uint32_t &Len, FileChecksumEntry &Item) {
Error llvm::VarStreamArrayExtractor<FileChecksumEntry>::
operator()(BinaryStreamRef Stream, uint32_t &Len, FileChecksumEntry &Item) {
BinaryStreamReader Reader(Stream);
const FileChecksumEntryHeader *Header;

6
llvm/lib/DebugInfo/CodeView/DebugCrossImpSubsection.cpp
View File

@ -16,9 +16,9 @@ using namespace llvm;
using namespace llvm::codeview;
namespace llvm {
Error VarStreamArrayExtractor<CrossModuleImportItem>::extract(
BinaryStreamRef Stream, uint32_t &Len,
codeview::CrossModuleImportItem &Item) {
Error VarStreamArrayExtractor<CrossModuleImportItem>::
operator()(BinaryStreamRef Stream, uint32_t &Len,
codeview::CrossModuleImportItem &Item) {
BinaryStreamReader Reader(Stream);
if (Reader.bytesRemaining() < sizeof(CrossModuleImport))
return make_error<CodeViewError>(

9
llvm/lib/DebugInfo/CodeView/DebugInlineeLinesSubsection.cpp
View File

@ -17,9 +17,8 @@
using namespace llvm;
using namespace llvm::codeview;
Error VarStreamArrayExtractor<InlineeSourceLine>::extract(
BinaryStreamRef Stream, uint32_t &Len, InlineeSourceLine &Item,
bool HasExtraFiles) {
Error VarStreamArrayExtractor<InlineeSourceLine>::
operator()(BinaryStreamRef Stream, uint32_t &Len, InlineeSourceLine &Item) {
BinaryStreamReader Reader(Stream);
if (auto EC = Reader.readObject(Item.Header))
@ -44,8 +43,8 @@ Error DebugInlineeLinesSubsectionRef::initialize(BinaryStreamReader Reader) {
if (auto EC = Reader.readEnum(Signature))
return EC;
if (auto EC =
Reader.readArray(Lines, Reader.bytesRemaining(), hasExtraFiles()))
Lines.getExtractor().HasExtraFiles = hasExtraFiles();
if (auto EC = Reader.readArray(Lines, Reader.bytesRemaining()))
return EC;
assert(Reader.bytesRemaining() == 0);

9
llvm/lib/DebugInfo/CodeView/DebugLinesSubsection.cpp
View File

@ -17,9 +17,8 @@
using namespace llvm;
using namespace llvm::codeview;
Error LineColumnExtractor::extract(BinaryStreamRef Stream, uint32_t &Len,
LineColumnEntry &Item,
const LineFragmentHeader *Header) {
Error LineColumnExtractor::operator()(BinaryStreamRef Stream, uint32_t &Len,
LineColumnEntry &Item) {
using namespace codeview;
const LineBlockFragmentHeader *BlockHeader;
BinaryStreamReader Reader(Stream);
@ -56,8 +55,8 @@ Error DebugLinesSubsectionRef::initialize(BinaryStreamReader Reader) {
if (auto EC = Reader.readObject(Header))
return EC;
if (auto EC =
Reader.readArray(LinesAndColumns, Reader.bytesRemaining(), Header))
LinesAndColumns.getExtractor().Header = Header;
if (auto EC = Reader.readArray(LinesAndColumns, Reader.bytesRemaining()))
return EC;
return Error::success();

3
llvm/lib/DebugInfo/CodeView/TypeTableCollection.cpp
View File

@ -51,7 +51,8 @@ void TypeTableCollection::ensureTypeExists(TypeIndex Index) {
CVType Type;
uint32_t Len;
error(VarStreamArrayExtractor<CVType>::extract(Bytes, Len, Type));
VarStreamArrayExtractor<CVType> Extract;
error(Extract(Bytes, Len, Type));
TypeDatabaseVisitor DBV(Database);
error(codeview::visitTypeRecord(Type, Index, DBV));

9
llvm/unittests/Support/BinaryStreamTest.cpp
View File

@ -416,9 +416,7 @@ TEST_F(BinaryStreamTest, VarStreamArray) {
struct StringExtractor {
public:
typedef uint32_t &ContextType;
static Error extract(BinaryStreamRef Stream, uint32_t &Len, StringRef &Item,
uint32_t &Index) {
Error operator()(BinaryStreamRef Stream, uint32_t &Len, StringRef &Item) {
if (Index == 0)
Len = strlen("1. Test");
else if (Index == 1)
@ -435,11 +433,12 @@ TEST_F(BinaryStreamTest, VarStreamArray) {
++Index;
return Error::success();
}
uint32_t Index = 0;
};
for (auto &Stream : Streams) {
uint32_t Context = 0;
VarStreamArray<StringRef, StringExtractor> Array(*Stream.Input, Context);
VarStreamArray<StringRef, StringExtractor> Array(*Stream.Input);
auto Iter = Array.begin();
ASSERT_EQ("1. Test", *Iter++);
ASSERT_EQ("2. Longer Test", *Iter++);