[ObjectYAML] Add offloading binary implementations for obj2yaml and yaml2obj

This patchs adds the necessary code for inspecting or creating offloading
binaries using the standing `obj2yaml` and `yaml2obj` features in LLVM.

Depends on D127774

Reviewed By: jhenderson

Differential Revision: https://reviews.llvm.org/D127776
This commit is contained in:
Joseph Huber 2022-06-14 15:04:39 -04:00
parent 927156a674
commit 228c8f9cc0
21 changed files with 508 additions and 4 deletions

View File

@ -103,7 +103,6 @@ public:
static bool classof(const Binary *V) { return V->isOffloadFile(); }
private:
struct Header {
uint8_t Magic[4] = {0x10, 0xFF, 0x10, 0xAD}; // 0x10FF10AD magic bytes.
uint32_t Version = OffloadBinary::Version; // Version identifier.
@ -127,6 +126,7 @@ private:
uint64_t ValueOffset;
};
private:
OffloadBinary(MemoryBufferRef Source, const Header *TheHeader,
const Entry *TheEntry)
: Binary(Binary::ID_Offload, Source), Buffer(Source.getBufferStart()),

View File

@ -15,6 +15,7 @@
#include "llvm/ObjectYAML/ELFYAML.h"
#include "llvm/ObjectYAML/MachOYAML.h"
#include "llvm/ObjectYAML/MinidumpYAML.h"
#include "llvm/ObjectYAML/OffloadYAML.h"
#include "llvm/ObjectYAML/WasmYAML.h"
#include "llvm/ObjectYAML/XCOFFYAML.h"
#include "llvm/Support/YAMLTraits.h"
@ -32,6 +33,7 @@ struct YamlObjectFile {
std::unique_ptr<MachOYAML::Object> MachO;
std::unique_ptr<MachOYAML::UniversalBinary> FatMachO;
std::unique_ptr<MinidumpYAML::Object> Minidump;
std::unique_ptr<OffloadYAML::Binary> Offload;
std::unique_ptr<WasmYAML::Object> Wasm;
std::unique_ptr<XCOFFYAML::Object> Xcoff;
std::unique_ptr<DXContainerYAML::Object> DXContainer;

View File

@ -0,0 +1,79 @@
//===- OffloadYAML.h - Offload Binary YAMLIO implementation -----*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
///
/// \file
/// This file declares classes for handling the YAML representation of
/// offloading binaries.
///
//===----------------------------------------------------------------------===//
#ifndef LLVM_OBJECTYAML_OFFLOADYAML_H
#define LLVM_OBJECTYAML_OFFLOADYAML_H
#include "llvm/ADT/MapVector.h"
#include "llvm/Object/OffloadBinary.h"
#include "llvm/ObjectYAML/YAML.h"
#include "llvm/Support/YAMLTraits.h"
namespace llvm {
namespace OffloadYAML {
struct Binary {
struct StringEntry {
StringRef Key;
StringRef Value;
};
struct Member {
Optional<object::ImageKind> ImageKind;
Optional<object::OffloadKind> OffloadKind;
Optional<uint32_t> Flags;
Optional<std::vector<StringEntry>> StringEntries;
Optional<yaml::BinaryRef> Content;
};
Optional<uint32_t> Version;
Optional<uint64_t> Size;
Optional<uint64_t> EntryOffset;
Optional<uint64_t> EntrySize;
std::vector<Member> Members;
};
} // end namespace OffloadYAML
} // end namespace llvm
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::OffloadYAML::Binary::Member)
LLVM_YAML_IS_SEQUENCE_VECTOR(llvm::OffloadYAML::Binary::StringEntry)
namespace llvm {
namespace yaml {
template <> struct ScalarEnumerationTraits<object::ImageKind> {
static void enumeration(IO &IO, object::ImageKind &Value);
};
template <> struct ScalarEnumerationTraits<object::OffloadKind> {
static void enumeration(IO &IO, object::OffloadKind &Value);
};
template <> struct MappingTraits<OffloadYAML::Binary> {
static void mapping(IO &IO, OffloadYAML::Binary &O);
};
template <> struct MappingTraits<OffloadYAML::Binary::StringEntry> {
static void mapping(IO &IO, OffloadYAML::Binary::StringEntry &M);
};
template <> struct MappingTraits<OffloadYAML::Binary::Member> {
static void mapping(IO &IO, OffloadYAML::Binary::Member &M);
};
} // end namespace yaml
} // end namespace llvm
#endif // LLVM_OBJECTYAML_ARCHIVEYAML_H

View File

@ -36,6 +36,10 @@ namespace MinidumpYAML {
struct Object;
}
namespace OffloadYAML {
struct Binary;
}
namespace WasmYAML {
struct Object;
}
@ -65,6 +69,7 @@ bool yaml2elf(ELFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH,
bool yaml2macho(YamlObjectFile &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2minidump(MinidumpYAML::Object &Doc, raw_ostream &Out,
ErrorHandler EH);
bool yaml2offload(OffloadYAML::Binary &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2wasm(WasmYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2xcoff(XCOFFYAML::Object &Doc, raw_ostream &Out, ErrorHandler EH);
bool yaml2dxcontainer(DXContainerYAML::Object &Doc, raw_ostream &Out,

View File

@ -18,6 +18,8 @@ add_llvm_component_library(LLVMObjectYAML
ObjectYAML.cpp
MinidumpEmitter.cpp
MinidumpYAML.cpp
OffloadEmitter.cpp
OffloadYAML.cpp
WasmEmitter.cpp
WasmYAML.cpp
XCOFFEmitter.cpp

View File

@ -56,6 +56,9 @@ void MappingTraits<YamlObjectFile>::mapping(IO &IO,
} else if (IO.mapTag("!minidump")) {
ObjectFile.Minidump.reset(new MinidumpYAML::Object());
MappingTraits<MinidumpYAML::Object>::mapping(IO, *ObjectFile.Minidump);
} else if (IO.mapTag("!Offload")) {
ObjectFile.Offload.reset(new OffloadYAML::Binary());
MappingTraits<OffloadYAML::Binary>::mapping(IO, *ObjectFile.Offload);
} else if (IO.mapTag("!WASM")) {
ObjectFile.Wasm.reset(new WasmYAML::Object());
MappingTraits<WasmYAML::Object>::mapping(IO, *ObjectFile.Wasm);

View File

@ -0,0 +1,68 @@
//===- OffloadEmitter.cpp -------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "llvm/Object/OffloadBinary.h"
#include "llvm/ObjectYAML/OffloadYAML.h"
#include "llvm/ObjectYAML/yaml2obj.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/raw_ostream.h"
using namespace llvm;
using namespace OffloadYAML;
namespace llvm {
namespace yaml {
bool yaml2offload(Binary &Doc, raw_ostream &Out, ErrorHandler EH) {
for (const auto &Member : Doc.Members) {
object::OffloadBinary::OffloadingImage Image{};
if (Member.ImageKind)
Image.TheImageKind = *Member.ImageKind;
if (Member.OffloadKind)
Image.TheOffloadKind = *Member.OffloadKind;
if (Member.Flags)
Image.Flags = *Member.Flags;
StringMap<StringRef> &StringData = Image.StringData;
if (Member.StringEntries) {
for (const auto &Entry : *Member.StringEntries) {
StringData[Entry.Key] = Entry.Value;
}
}
SmallVector<char, 1024> Data;
raw_svector_ostream OS(Data);
if (Member.Content)
Member.Content->writeAsBinary(OS);
Image.Image = MemoryBuffer::getMemBufferCopy(OS.str());
std::unique_ptr<MemoryBuffer> Binary = object::OffloadBinary::write(Image);
// Copy the data to a new buffer so we can modify the bytes directly.
SmallVector<char> NewBuffer;
std::copy(Binary->getBufferStart(), Binary->getBufferEnd(),
std::back_inserter(NewBuffer));
auto *TheHeader =
reinterpret_cast<object::OffloadBinary::Header *>(&NewBuffer[0]);
if (Doc.Version)
TheHeader->Version = *Doc.Version;
if (Doc.Size)
TheHeader->Size = *Doc.Size;
if (Doc.EntryOffset)
TheHeader->EntryOffset = *Doc.EntryOffset;
if (Doc.EntrySize)
TheHeader->EntrySize = *Doc.EntrySize;
Out.write(NewBuffer.begin(), NewBuffer.size());
}
return true;
}
} // namespace yaml
} // namespace llvm

View File

@ -0,0 +1,78 @@
//===- OffloadYAML.cpp - Offload Binary YAMLIO implementation -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines classes for handling the YAML representation of offload
// binaries.
//
//===----------------------------------------------------------------------===//
#include <llvm/ObjectYAML/OffloadYAML.h>
namespace llvm {
namespace yaml {
void ScalarEnumerationTraits<object::ImageKind>::enumeration(
IO &IO, object::ImageKind &Value) {
#define ECase(X) IO.enumCase(Value, #X, object::X)
ECase(IMG_None);
ECase(IMG_Object);
ECase(IMG_Bitcode);
ECase(IMG_Cubin);
ECase(IMG_Fatbinary);
ECase(IMG_PTX);
ECase(IMG_LAST);
#undef ECase
IO.enumFallback<Hex16>(Value);
}
void ScalarEnumerationTraits<object::OffloadKind>::enumeration(
IO &IO, object::OffloadKind &Value) {
#define ECase(X) IO.enumCase(Value, #X, object::X)
ECase(OFK_None);
ECase(OFK_OpenMP);
ECase(OFK_Cuda);
ECase(OFK_HIP);
ECase(OFK_LAST);
#undef ECase
IO.enumFallback<Hex16>(Value);
}
void MappingTraits<OffloadYAML::Binary>::mapping(IO &IO,
OffloadYAML::Binary &O) {
assert(!IO.getContext() && "The IO context is initialized already");
IO.setContext(&O);
IO.mapTag("!Offload", true);
IO.mapOptional("Version", O.Version);
IO.mapOptional("Size", O.Size);
IO.mapOptional("EntryOffset", O.EntryOffset);
IO.mapOptional("EntrySize", O.EntrySize);
IO.mapRequired("Members", O.Members);
IO.setContext(nullptr);
}
void MappingTraits<OffloadYAML::Binary::StringEntry>::mapping(
IO &IO, OffloadYAML::Binary::StringEntry &SE) {
assert(IO.getContext() && "The IO context is not initialized");
IO.mapRequired("Key", SE.Key);
IO.mapRequired("Value", SE.Value);
}
void MappingTraits<OffloadYAML::Binary::Member>::mapping(
IO &IO, OffloadYAML::Binary::Member &M) {
assert(IO.getContext() && "The IO context is not initialized");
IO.mapOptional("ImageKind", M.ImageKind);
IO.mapOptional("OffloadKind", M.OffloadKind);
IO.mapOptional("Flags", M.Flags);
IO.mapOptional("String", M.StringEntries);
IO.mapOptional("Content", M.Content);
}
} // namespace yaml
} // namespace llvm

View File

@ -42,6 +42,8 @@ bool convertYAML(yaml::Input &YIn, raw_ostream &Out, ErrorHandler ErrHandler,
return yaml2macho(Doc, Out, ErrHandler);
if (Doc.Minidump)
return yaml2minidump(*Doc.Minidump, Out, ErrHandler);
if (Doc.Offload)
return yaml2offload(*Doc.Offload, Out, ErrHandler);
if (Doc.Wasm)
return yaml2wasm(*Doc.Wasm, Out, ErrHandler);
if (Doc.Xcoff)

View File

@ -0,0 +1,25 @@
# RUN: yaml2obj %s | obj2yaml | FileCheck %s
!Offload
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
# CHECK: --- !Offload
# CHECK-NEXT: Members:
# CHECK-NEXT: - ImageKind: IMG_Cubin
# CHECK-NEXT: OffloadKind: OFK_OpenMP
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: String:
# CHECK-NEXT: - Key: triple
# CHECK-NEXT: Value: nvptx64-nvidia-cuda
# CHECK-NEXT: - Key: arch
# CHECK-NEXT: Value: sm_70
# CHECK-NEXT: Content: DEADBEEF
# CHECK-NEXT: ...

View File

@ -0,0 +1,11 @@
# RUN: yaml2obj %s | obj2yaml | FileCheck %s
!Offload
Members:
-
# CHECK: --- !Offload
# CHECK-NEXT: Members:
# CHECK-NEXT: - ImageKind: IMG_None
# CHECK-NEXT: OffloadKind: OFK_None
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: ...

View File

@ -0,0 +1,40 @@
# RUN: yaml2obj %s | obj2yaml | FileCheck %s
!Offload
Members:
- ImageKind: IMG_None
OffloadKind: OFK_None
- ImageKind: IMG_Object
OffloadKind: OFK_OpenMP
- ImageKind: IMG_Bitcode
OffloadKind: OFK_Cuda
- ImageKind: IMG_Cubin
OffloadKind: OFK_HIP
- ImageKind: IMG_Fatbinary
- ImageKind: IMG_PTX
- ImageKind: 999
OffloadKind: 999
# CHECK: --- !Offload
# CHECK-NEXT: Members:
# CHECK-NEXT: - ImageKind: IMG_None
# CHECK-NEXT: OffloadKind: OFK_None
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: IMG_Object
# CHECK-NEXT: OffloadKind: OFK_OpenMP
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: IMG_Bitcode
# CHECK-NEXT: OffloadKind: OFK_Cuda
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: IMG_Cubin
# CHECK-NEXT: OffloadKind: OFK_HIP
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: IMG_Fatbinary
# CHECK-NEXT: OffloadKind: OFK_None
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: IMG_PTX
# CHECK-NEXT: OffloadKind: OFK_None
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: - ImageKind: 0x3E7
# CHECK-NEXT: OffloadKind: 0x3E7
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: ...

View File

@ -0,0 +1,15 @@
# RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s
!Offload
EntrySize: 999999999
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
# CHECK: Error reading file: <stdin>: The end of the file was unexpectedly encountered

View File

@ -0,0 +1,15 @@
# RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s
!Offload
EntryOffset: 999999999
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
# CHECK: Error reading file: <stdin>: The end of the file was unexpectedly encountered

View File

@ -0,0 +1,15 @@
# RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s
!Offload
Size: 999999999
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
# CHECK: Error reading file: <stdin>: The end of the file was unexpectedly encountered

View File

@ -0,0 +1,15 @@
# RUN: yaml2obj %s | not obj2yaml 2>&1 | FileCheck %s
!Offload
Version: 2
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
# CHECK: Error reading file: <stdin>: Invalid data was encountered while parsing the file

View File

@ -0,0 +1,43 @@
# RUN: yaml2obj %s | obj2yaml | FileCheck %s
!Offload
Members:
- ImageKind: IMG_Cubin
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "nvptx64-nvidia-cuda"
- Key: "arch"
Value: "sm_70"
Content: "deadbeef"
- ImageKind: IMG_Bitcode
OffloadKind: OFK_OpenMP
Flags: 0
String:
- Key: "triple"
Value: "amdgcn-amd-amdhsa"
- Key: "arch"
Value: "gfx908"
Content: "cafefeed"
# CHECK: --- !Offload
# CHECK-NEXT: Members:
# CHECK-NEXT: - ImageKind: IMG_Cubin
# CHECK-NEXT: OffloadKind: OFK_OpenMP
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: String:
# CHECK-NEXT: - Key: triple
# CHECK-NEXT: Value: nvptx64-nvidia-cuda
# CHECK-NEXT: - Key: arch
# CHECK-NEXT: Value: sm_70
# CHECK-NEXT: Content: DEADBEEF
# CHECK-NEXT: - ImageKind: IMG_Bitcode
# CHECK-NEXT: OffloadKind: OFK_OpenMP
# CHECK-NEXT: Flags: 0
# CHECK-NEXT: String:
# CHECK-NEXT: - Key: triple
# CHECK-NEXT: Value: amdgcn-amd-amdhsa
# CHECK-NEXT: - Key: arch
# CHECK-NEXT: Value: gfx908
# CHECK-NEXT: Content: CAFEFEED
# CHECK-NEXT: ...

View File

@ -16,6 +16,7 @@ add_llvm_utility(obj2yaml
elf2yaml.cpp
macho2yaml.cpp
minidump2yaml.cpp
offload2yaml.cpp
xcoff2yaml.cpp
wasm2yaml.cpp
)

View File

@ -55,6 +55,8 @@ static Error dumpInput(StringRef File) {
return archive2yaml(outs(), MemBuf);
case file_magic::dxcontainer_object:
return dxcontainer2yaml(outs(), MemBuf);
case file_magic::offload_binary:
return offload2yaml(outs(), MemBuf);
default:
break;
}

View File

@ -16,8 +16,8 @@
#include "llvm/Object/Minidump.h"
#include "llvm/Object/Wasm.h"
#include "llvm/Object/XCOFFObjectFile.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/raw_ostream.h"
#include <system_error>
enum RawSegments : unsigned { none = 0, data = 1, linkedit = 1 << 1 };
@ -34,6 +34,7 @@ llvm::Error xcoff2yaml(llvm::raw_ostream &Out,
std::error_code wasm2yaml(llvm::raw_ostream &Out,
const llvm::object::WasmObjectFile &Obj);
llvm::Error archive2yaml(llvm::raw_ostream &Out, llvm::MemoryBufferRef Source);
llvm::Error offload2yaml(llvm::raw_ostream &Out, llvm::MemoryBufferRef Source);
llvm::Error dxcontainer2yaml(llvm::raw_ostream &Out,
llvm::MemoryBufferRef Source);
@ -43,7 +44,7 @@ class DWARFContext;
namespace DWARFYAML {
struct Data;
}
}
} // namespace llvm
void dumpDebugAbbrev(llvm::DWARFContext &DCtx, llvm::DWARFYAML::Data &Y);
llvm::Error dumpDebugAddr(llvm::DWARFContext &DCtx, llvm::DWARFYAML::Data &Y);

View File

@ -0,0 +1,82 @@
//===------ offload2yaml.cpp - obj2yaml conversion tool ---*- C++ -------*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "obj2yaml.h"
#include "llvm/BinaryFormat/Magic.h"
#include "llvm/Object/OffloadBinary.h"
#include "llvm/ObjectYAML/OffloadYAML.h"
#include "llvm/Support/StringSaver.h"
using namespace llvm;
namespace {
void populateYAML(OffloadYAML::Binary &YAMLBinary, object::OffloadBinary &OB,
UniqueStringSaver Saver) {
YAMLBinary.Members.emplace_back();
auto &Member = YAMLBinary.Members.back();
Member.ImageKind = OB.getImageKind();
Member.OffloadKind = OB.getOffloadKind();
Member.Flags = OB.getFlags();
if (!OB.strings().empty()) {
Member.StringEntries = std::vector<OffloadYAML::Binary::StringEntry>();
for (const auto &Entry : OB.strings())
Member.StringEntries->emplace_back(OffloadYAML::Binary::StringEntry(
{Saver.save(Entry.getKey()), Saver.save(Entry.getValue())}));
}
if (!OB.getImage().empty())
Member.Content = arrayRefFromStringRef(OB.getImage());
}
Expected<OffloadYAML::Binary *> dump(MemoryBufferRef Source,
UniqueStringSaver Saver) {
Expected<std::unique_ptr<object::OffloadBinary>> OB =
object::OffloadBinary::create(Source);
if (!OB)
return OB.takeError();
std::unique_ptr<OffloadYAML::Binary> YAMLBinary =
std::make_unique<OffloadYAML::Binary>();
YAMLBinary->Members = std::vector<OffloadYAML::Binary::Member>();
uint64_t Offset = 0;
while (Offset < (*OB)->getMemoryBufferRef().getBufferSize()) {
MemoryBufferRef Buffer = MemoryBufferRef(
(*OB)->getData().drop_front(Offset), (*OB)->getFileName());
auto BinaryOrErr = object::OffloadBinary::create(Buffer);
if (!BinaryOrErr)
return BinaryOrErr.takeError();
object::OffloadBinary &Binary = **BinaryOrErr;
populateYAML(*YAMLBinary, Binary, Saver);
Offset += Binary.getSize();
}
return YAMLBinary.release();
}
} // namespace
Error offload2yaml(raw_ostream &Out, MemoryBufferRef Source) {
BumpPtrAllocator Alloc;
UniqueStringSaver Saver(Alloc);
Expected<OffloadYAML::Binary *> YAMLOrErr = dump(Source, Saver);
if (!YAMLOrErr)
return YAMLOrErr.takeError();
std::unique_ptr<OffloadYAML::Binary> YAML(YAMLOrErr.get());
yaml::Output Yout(Out);
Yout << *YAML;
return Error::success();
}