MC: Add support for BigObj

Teach WinCOFFObjectWriter how to write -mbig-obj style object files;
these object files allow for more sections inside an object file.

Our support for BigObj is notably different from binutils and cl: we
implicitly upgrade object files to BigObj instead of asking the user to
compile the same file *again* but with another flag.  This matches up
with how LLVM treats ELF variants.

This was tested by forcing LLVM to always emit BigObj files and running
the entire test suite.  A specific test has also been added.

I've lowered the maximum number of sections in a normal COFF file,
VS "14" CTP 3 supports no more than 65279 sections.  This is important
otherwise we might not switch to BigObj quickly enough, leaving us with
a COFF file that we couldn't link.

yaml2obj support is all that remains to implement.

Differential Revision: http://reviews.llvm.org/D5349

llvm-svn: 217812

llvm/include/llvm/Object/COFF.h

@@ -236,6 +236,14 @@ public:
     return A.getRawPtr() < B.getRawPtr();
   }
 
+  bool isBigObj() const {
+    if (CS16)
+      return false;
+    if (CS32)
+      return true;
+    llvm_unreachable("COFFSymbolRef points to nothing!");
+  }
+
   const char *getShortName() const {
     return CS16 ? CS16->Name.ShortName : CS32->Name.ShortName;
   }
@@ -361,8 +369,16 @@ struct coff_aux_section_definition {
   support::ulittle16_t NumberOfRelocations;
   support::ulittle16_t NumberOfLinenumbers;
   support::ulittle32_t CheckSum;
-  support::ulittle16_t Number;
+  support::ulittle16_t NumberLowPart;
   uint8_t              Selection;
+  uint8_t              Unused;
+  support::ulittle16_t NumberHighPart;
+  int32_t getNumber(bool IsBigObj) const {
+    uint32_t Number = static_cast<uint32_t>(NumberLowPart);
+    if (IsBigObj)
+      Number |= static_cast<uint32_t>(NumberHighPart) << 16;
+    return static_cast<int32_t>(Number);
+  }
 };
 
 struct coff_aux_clr_token {

llvm/include/llvm/Support/COFF.h

@@ -31,7 +31,7 @@ namespace llvm {
 namespace COFF {
 
   // The maximum number of sections that a COFF object can have (inclusive).
-  const uint16_t MaxNumberOfSections16 = 65299;
+  const int32_t MaxNumberOfSections16 = 65279;
 
   // The PE signature bytes that follows the DOS stub header.
   static const char PEMagic[] = { 'P', 'E', '\0', '\0' };
@@ -43,16 +43,18 @@ namespace COFF {
 
   // Sizes in bytes of various things in the COFF format.
   enum {
-    HeaderSize     = 20,
+    Header16Size   = 20,
+    Header32Size   = 56,
     NameSize       = 8,
-    SymbolSize     = 18,
+    Symbol16Size   = 18,
+    Symbol32Size   = 20,
     SectionSize    = 40,
     RelocationSize = 10
   };
 
   struct header {
     uint16_t Machine;
-    uint16_t NumberOfSections;
+    int32_t  NumberOfSections;
     uint32_t TimeDateStamp;
     uint32_t PointerToSymbolTable;
     uint32_t NumberOfSymbols;
@@ -147,7 +149,7 @@ namespace COFF {
   struct symbol {
     char     Name[NameSize];
     uint32_t Value;
-    uint16_t SectionNumber;
+    int32_t  SectionNumber;
     uint16_t Type;
     uint8_t  StorageClass;
     uint8_t  NumberOfAuxSymbols;
@@ -390,18 +392,14 @@ namespace COFF {
     IMAGE_WEAK_EXTERN_SEARCH_ALIAS     = 3
   };
 
-  struct AuxiliaryFile {
-    uint8_t FileName[18];
-  };
-
   struct AuxiliarySectionDefinition {
     uint32_t Length;
     uint16_t NumberOfRelocations;
     uint16_t NumberOfLinenumbers;
     uint32_t CheckSum;
-    uint16_t Number;
+    uint32_t Number;
     uint8_t  Selection;
-    char     unused[3];
+    char     unused;
   };
 
   struct AuxiliaryCLRToken {
@@ -415,7 +413,6 @@ namespace COFF {
     AuxiliaryFunctionDefinition FunctionDefinition;
     AuxiliarybfAndefSymbol      bfAndefSymbol;
     AuxiliaryWeakExternal       WeakExternal;
-    AuxiliaryFile               File;
     AuxiliarySectionDefinition  SectionDefinition;
   };
 

llvm/lib/MC/WinCOFFObjectWriter.cpp

@@ -71,7 +71,6 @@ public:
   MCSymbolData const *MCData;
 
   COFFSymbol(StringRef name);
-  size_t size() const;
   void set_name_offset(uint32_t Offset);
 
   bool should_keep() const;
@@ -137,6 +136,8 @@ public:
   section_map SectionMap;
   symbol_map  SymbolMap;
 
+  bool UseBigObj;
+
   WinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW, raw_ostream &OS);
 
   COFFSymbol *createSymbol(StringRef Name);
@@ -199,10 +200,6 @@ COFFSymbol::COFFSymbol(StringRef name)
   memset(&Data, 0, sizeof(Data));
 }
 
-size_t COFFSymbol::size() const {
-  return COFF::SymbolSize + (Data.NumberOfAuxSymbols * COFF::SymbolSize);
-}
-
 // In the case that the name does not fit within 8 bytes, the offset
 // into the string table is stored in the last 4 bytes instead, leaving
 // the first 4 bytes as 0.
@@ -301,8 +298,7 @@ size_t StringTable::insert(StringRef String) {
 
 WinCOFFObjectWriter::WinCOFFObjectWriter(MCWinCOFFObjectTargetWriter *MOTW,
                                          raw_ostream &OS)
-  : MCObjectWriter(OS, true)
+    : MCObjectWriter(OS, true), TargetObjectWriter(MOTW) {
-  , TargetObjectWriter(MOTW) {
   memset(&Header, 0, sizeof(Header));
 
   Header.Machine = TargetObjectWriter->getMachine();
@@ -578,19 +574,39 @@ bool WinCOFFObjectWriter::IsPhysicalSection(COFFSection *S) {
 // entity writing methods
 
 void WinCOFFObjectWriter::WriteFileHeader(const COFF::header &Header) {
+  if (UseBigObj) {
+    WriteLE16(COFF::IMAGE_FILE_MACHINE_UNKNOWN);
+    WriteLE16(0xFFFF);
+    WriteLE16(COFF::BigObjHeader::MinBigObjectVersion);
     WriteLE16(Header.Machine);
-  WriteLE16(Header.NumberOfSections);
+    WriteLE32(Header.TimeDateStamp);
+    for (uint8_t MagicChar : COFF::BigObjMagic)
+      Write8(MagicChar);
+    WriteZeros(sizeof(COFF::BigObjHeader::unused1));
+    WriteZeros(sizeof(COFF::BigObjHeader::unused2));
+    WriteZeros(sizeof(COFF::BigObjHeader::unused3));
+    WriteZeros(sizeof(COFF::BigObjHeader::unused4));
+    WriteLE32(Header.NumberOfSections);
+    WriteLE32(Header.PointerToSymbolTable);
+    WriteLE32(Header.NumberOfSymbols);
+  } else {
+    WriteLE16(Header.Machine);
+    WriteLE16(static_cast<int16_t>(Header.NumberOfSections));
     WriteLE32(Header.TimeDateStamp);
     WriteLE32(Header.PointerToSymbolTable);
     WriteLE32(Header.NumberOfSymbols);
     WriteLE16(Header.SizeOfOptionalHeader);
     WriteLE16(Header.Characteristics);
   }
+}
 
 void WinCOFFObjectWriter::WriteSymbol(const COFFSymbol &S) {
   WriteBytes(StringRef(S.Data.Name, COFF::NameSize));
   WriteLE32(S.Data.Value);
-  WriteLE16(S.Data.SectionNumber);
+  if (UseBigObj)
+    WriteLE32(S.Data.SectionNumber);
+  else
+    WriteLE16(static_cast<int16_t>(S.Data.SectionNumber));
   WriteLE16(S.Data.Type);
   Write8(S.Data.StorageClass);
   Write8(S.Data.NumberOfAuxSymbols);
@@ -608,6 +624,8 @@ void WinCOFFObjectWriter::WriteAuxiliarySymbols(
       WriteLE32(i->Aux.FunctionDefinition.PointerToLinenumber);
       WriteLE32(i->Aux.FunctionDefinition.PointerToNextFunction);
       WriteZeros(sizeof(i->Aux.FunctionDefinition.unused));
+      if (UseBigObj)
+        WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATbfAndefSymbol:
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused1));
@@ -615,24 +633,32 @@ void WinCOFFObjectWriter::WriteAuxiliarySymbols(
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused2));
       WriteLE32(i->Aux.bfAndefSymbol.PointerToNextFunction);
       WriteZeros(sizeof(i->Aux.bfAndefSymbol.unused3));
+      if (UseBigObj)
+        WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATWeakExternal:
       WriteLE32(i->Aux.WeakExternal.TagIndex);
       WriteLE32(i->Aux.WeakExternal.Characteristics);
       WriteZeros(sizeof(i->Aux.WeakExternal.unused));
+      if (UseBigObj)
+        WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     case ATFile:
-      WriteBytes(StringRef(reinterpret_cast<const char *>(i->Aux.File.FileName),
+      WriteBytes(
-                 sizeof(i->Aux.File.FileName)));
+          StringRef(reinterpret_cast<const char *>(&i->Aux),
+                    UseBigObj ? COFF::Symbol32Size : COFF::Symbol16Size));
       break;
     case ATSectionDefinition:
       WriteLE32(i->Aux.SectionDefinition.Length);
       WriteLE16(i->Aux.SectionDefinition.NumberOfRelocations);
       WriteLE16(i->Aux.SectionDefinition.NumberOfLinenumbers);
       WriteLE32(i->Aux.SectionDefinition.CheckSum);
-      WriteLE16(i->Aux.SectionDefinition.Number);
+      WriteLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number));
       Write8(i->Aux.SectionDefinition.Selection);
       WriteZeros(sizeof(i->Aux.SectionDefinition.unused));
+      WriteLE16(static_cast<int16_t>(i->Aux.SectionDefinition.Number >> 16));
+      if (UseBigObj)
+        WriteZeros(COFF::Symbol32Size - COFF::Symbol16Size);
       break;
     }
   }
@@ -665,37 +691,6 @@ void WinCOFFObjectWriter::ExecutePostLayoutBinding(MCAssembler &Asm,
                                                    const MCAsmLayout &Layout) {
   // "Define" each section & symbol. This creates section & symbol
   // entries in the staging area.
-
-  static_assert(sizeof(((COFF::AuxiliaryFile *)nullptr)->FileName) == COFF::SymbolSize,
-                "size mismatch for COFF::AuxiliaryFile::FileName");
-  for (auto FI = Asm.file_names_begin(), FE = Asm.file_names_end();
-       FI != FE; ++FI) {
-    // round up to calculate the number of auxiliary symbols required
-    unsigned Count = (FI->size() + COFF::SymbolSize - 1) / COFF::SymbolSize;
-
-    COFFSymbol *file = createSymbol(".file");
-    file->Data.SectionNumber = COFF::IMAGE_SYM_DEBUG;
-    file->Data.StorageClass = COFF::IMAGE_SYM_CLASS_FILE;
-    file->Aux.resize(Count);
-
-    unsigned Offset = 0;
-    unsigned Length = FI->size();
-    for (auto & Aux : file->Aux) {
-      Aux.AuxType = ATFile;
-
-      if (Length > COFF::SymbolSize) {
-        memcpy(Aux.Aux.File.FileName, FI->c_str() + Offset, COFF::SymbolSize);
-        Length = Length - COFF::SymbolSize;
-      } else {
-        memcpy(Aux.Aux.File.FileName, FI->c_str() + Offset, Length);
-        memset(&Aux.Aux.File.FileName[Length], 0, COFF::SymbolSize - Length);
-        Length = 0;
-      }
-
-      Offset = Offset + COFF::SymbolSize;
-    }
-  }
-
   for (const auto & Section : Asm)
     DefineSection(Section);
 
@@ -839,23 +834,57 @@ void WinCOFFObjectWriter::RecordRelocation(const MCAssembler &Asm,
 
 void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
                                       const MCAsmLayout &Layout) {
-  // Assign symbol and section indexes and offsets.
+  size_t SectionsSize = Sections.size();
-  size_t NumberOfSections = 0;
+  if (SectionsSize > static_cast<size_t>(INT32_MAX))
+    report_fatal_error(
+        "PE COFF object files can't have more than 2147483647 sections");
 
-  DenseMap<COFFSection *, uint16_t> SectionIndices;
+  // Assign symbol and section indexes and offsets.
+  int32_t NumberOfSections = static_cast<int32_t>(SectionsSize);
+
+  UseBigObj = NumberOfSections > COFF::MaxNumberOfSections16;
+
+  DenseMap<COFFSection *, int32_t> SectionIndices(
+      NextPowerOf2(NumberOfSections));
+  size_t Number = 1;
   for (const auto &Section : Sections) {
-    size_t Number = ++NumberOfSections;
+    SectionIndices[Section.get()] = Number;
-    SectionIndices[Section.get()] = static_cast<uint16_t>(Number);
     MakeSectionReal(*Section, Number);
+    ++Number;
   }
 
-  if (NumberOfSections > static_cast<size_t>(COFF::MaxNumberOfSections16))
+  Header.NumberOfSections = NumberOfSections;
-    report_fatal_error(
-        "PE COFF object files can't have more than 65,299 sections");
-
-  Header.NumberOfSections = static_cast<uint16_t>(NumberOfSections);
   Header.NumberOfSymbols = 0;
 
+  for (auto FI = Asm.file_names_begin(), FE = Asm.file_names_end();
+       FI != FE; ++FI) {
+    // round up to calculate the number of auxiliary symbols required
+    unsigned SymbolSize = UseBigObj ? COFF::Symbol32Size : COFF::Symbol16Size;
+    unsigned Count = (FI->size() + SymbolSize - 1) / SymbolSize;
+
+    COFFSymbol *file = createSymbol(".file");
+    file->Data.SectionNumber = COFF::IMAGE_SYM_DEBUG;
+    file->Data.StorageClass = COFF::IMAGE_SYM_CLASS_FILE;
+    file->Aux.resize(Count);
+
+    unsigned Offset = 0;
+    unsigned Length = FI->size();
+    for (auto & Aux : file->Aux) {
+      Aux.AuxType = ATFile;
+
+      if (Length > SymbolSize) {
+        memcpy(&Aux.Aux, FI->c_str() + Offset, SymbolSize);
+        Length = Length - SymbolSize;
+      } else {
+        memcpy(&Aux.Aux, FI->c_str() + Offset, Length);
+        memset((char *)&Aux.Aux + Length, 0, SymbolSize - Length);
+        break;
+      }
+
+      Offset += SymbolSize;
+    }
+  }
+
   for (auto &Symbol : Symbols) {
     // Update section number & offset for symbols that have them.
     if (Symbol->Section)
@@ -913,7 +942,10 @@ void WinCOFFObjectWriter::WriteObject(MCAssembler &Asm,
 
   unsigned offset = 0;
 
-  offset += COFF::HeaderSize;
+  if (UseBigObj)
+    offset += COFF::Header32Size;
+  else
+    offset += COFF::Header16Size;
   offset += COFF::SectionSize * Header.NumberOfSections;
 
   for (const auto & Section : Asm) {

llvm/tools/llvm-objdump/llvm-objdump.cpp

@@ -591,6 +591,8 @@ static void PrintCOFFSymbolTable(const COFFObjectFile *coff) {
         if (error(coff->getAuxSymbol<coff_aux_section_definition>(SI + 1, asd)))
           return;
 
+        int32_t AuxNumber = asd->getNumber(Symbol->isBigObj());
+
         outs() << "AUX "
                << format("scnlen 0x%x nreloc %d nlnno %d checksum 0x%x "
                          , unsigned(asd->Length)
@@ -598,7 +600,7 @@ static void PrintCOFFSymbolTable(const COFFObjectFile *coff) {
                          , unsigned(asd->NumberOfLinenumbers)
                          , unsigned(asd->CheckSum))
                << format("assoc %d comdat %d\n"
-                         , unsigned(asd->Number)
+                         , unsigned(AuxNumber)
                          , unsigned(asd->Selection));
       } else if (Symbol->isFileRecord()) {
         const char *FileName;

llvm/tools/llvm-readobj/COFFDumper.cpp

@@ -790,12 +790,14 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
       if (error(getSymbolAuxData(Obj, Symbol, I, Aux)))
         break;
 
+      int32_t AuxNumber = Aux->getNumber(Symbol.isBigObj());
+
       DictScope AS(W, "AuxSectionDef");
       W.printNumber("Length", Aux->Length);
       W.printNumber("RelocationCount", Aux->NumberOfRelocations);
       W.printNumber("LineNumberCount", Aux->NumberOfLinenumbers);
       W.printHex("Checksum", Aux->CheckSum);
-      W.printNumber("Number", Aux->Number);
+      W.printNumber("Number", AuxNumber);
       W.printEnum("Selection", Aux->Selection, makeArrayRef(ImageCOMDATSelect));
 
       if (Section && Section->Characteristics & COFF::IMAGE_SCN_LNK_COMDAT
@@ -803,13 +805,13 @@ void COFFDumper::printSymbol(const SymbolRef &Sym) {
         const coff_section *Assoc;
         StringRef AssocName;
         std::error_code EC;
-        if ((EC = Obj->getSection(Aux->Number, Assoc)) ||
+        if ((EC = Obj->getSection(AuxNumber, Assoc)) ||
             (EC = Obj->getSectionName(Assoc, AssocName))) {
           AssocName = "";
           error(EC);
         }
 
-        W.printNumber("AssocSection", AssocName, Aux->Number);
+        W.printNumber("AssocSection", AssocName, AuxNumber);
       }
     } else if (Symbol.isCLRToken()) {
       const coff_aux_clr_token *Aux;

llvm/tools/obj2yaml/coff2yaml.cpp

@@ -104,13 +104,15 @@ static void dumpWeakExternal(COFFYAML::Symbol *Sym,
 
 static void
 dumpSectionDefinition(COFFYAML::Symbol *Sym,
-                      const object::coff_aux_section_definition *ObjSD) {
+                      const object::coff_aux_section_definition *ObjSD,
+                      bool IsBigObj) {
   COFF::AuxiliarySectionDefinition YAMLASD;
+  int32_t AuxNumber = ObjSD->getNumber(IsBigObj);
   YAMLASD.Length = ObjSD->Length;
   YAMLASD.NumberOfRelocations = ObjSD->NumberOfRelocations;
   YAMLASD.NumberOfLinenumbers = ObjSD->NumberOfLinenumbers;
   YAMLASD.CheckSum = ObjSD->CheckSum;
-  YAMLASD.Number = ObjSD->Number;
+  YAMLASD.Number = AuxNumber;
   YAMLASD.Selection = ObjSD->Selection;
 
   Sym->SectionDefinition = YAMLASD;
@@ -182,7 +184,7 @@ void COFFDumper::dumpSymbols(unsigned NumSymbols) {
         const object::coff_aux_section_definition *ObjSD =
             reinterpret_cast<const object::coff_aux_section_definition *>(
                 AuxData.data());
-        dumpSectionDefinition(&Sym, ObjSD);
+        dumpSectionDefinition(&Sym, ObjSD, Symbol.isBigObj());
       } else if (Symbol.isCLRToken()) {
         // This symbol represents a CLR token definition.
         assert(Symbol.getNumberOfAuxSymbols() == 1 &&

llvm/tools/yaml2obj/yaml2coff.cpp

@@ -121,8 +121,8 @@ static bool layoutCOFF(COFFParser &CP) {
 
   // The section table starts immediately after the header, including the
   // optional header.
-  SectionTableStart = sizeof(COFF::header) + CP.Obj.Header.SizeOfOptionalHeader;
+  SectionTableStart = COFF::Header16Size + CP.Obj.Header.SizeOfOptionalHeader;
-  SectionTableSize = sizeof(COFF::section) * CP.Obj.Sections.size();
+  SectionTableSize = COFF::SectionSize * CP.Obj.Sections.size();
 
   uint32_t CurrentSectionDataOffset = SectionTableStart + SectionTableSize;
 
@@ -163,7 +163,7 @@ static bool layoutCOFF(COFFParser &CP) {
       NumberOfAuxSymbols += 1;
     if (!i->File.empty())
       NumberOfAuxSymbols +=
-          (i->File.size() + COFF::SymbolSize - 1) / COFF::SymbolSize;
+          (i->File.size() + COFF::Symbol16Size - 1) / COFF::Symbol16Size;
     if (i->SectionDefinition)
       NumberOfAuxSymbols += 1;
     if (i->CLRToken)
@@ -224,7 +224,7 @@ zeros_impl<sizeof(T)> zeros(const T &) {
 
 bool writeCOFF(COFFParser &CP, raw_ostream &OS) {
   OS << binary_le(CP.Obj.Header.Machine)
-     << binary_le(CP.Obj.Header.NumberOfSections)
+     << binary_le(static_cast<int16_t>(CP.Obj.Header.NumberOfSections))
      << binary_le(CP.Obj.Header.TimeDateStamp)
      << binary_le(CP.Obj.Header.PointerToSymbolTable)
      << binary_le(CP.Obj.Header.NumberOfSymbols)
@@ -277,7 +277,7 @@ bool writeCOFF(COFFParser &CP, raw_ostream &OS) {
                                                      i != e; ++i) {
     OS.write(i->Header.Name, COFF::NameSize);
     OS << binary_le(i->Header.Value)
-       << binary_le(i->Header.SectionNumber)
+       << binary_le(static_cast<int16_t>(i->Header.SectionNumber))
        << binary_le(i->Header.Type)
        << binary_le(i->Header.StorageClass)
        << binary_le(i->Header.NumberOfAuxSymbols);
@@ -300,8 +300,8 @@ bool writeCOFF(COFFParser &CP, raw_ostream &OS) {
          << zeros(i->WeakExternal->unused);
     if (!i->File.empty()) {
       uint32_t NumberOfAuxRecords =
-          (i->File.size() + COFF::SymbolSize - 1) / COFF::SymbolSize;
+          (i->File.size() + COFF::Symbol16Size - 1) / COFF::Symbol16Size;
-      uint32_t NumberOfAuxBytes = NumberOfAuxRecords * COFF::SymbolSize;
+      uint32_t NumberOfAuxBytes = NumberOfAuxRecords * COFF::Symbol16Size;
       uint32_t NumZeros = NumberOfAuxBytes - i->File.size();
       OS.write(i->File.data(), i->File.size());
       for (uint32_t Padding = 0; Padding < NumZeros; ++Padding)
@@ -312,9 +312,10 @@ bool writeCOFF(COFFParser &CP, raw_ostream &OS) {
          << binary_le(i->SectionDefinition->NumberOfRelocations)
          << binary_le(i->SectionDefinition->NumberOfLinenumbers)
          << binary_le(i->SectionDefinition->CheckSum)
-         << binary_le(i->SectionDefinition->Number)
+         << binary_le(static_cast<int16_t>(i->SectionDefinition->Number))
          << binary_le(i->SectionDefinition->Selection)
-         << zeros(i->SectionDefinition->unused);
+         << zeros(i->SectionDefinition->unused)
+         << binary_le(static_cast<int16_t>(i->SectionDefinition->Number >> 16));
     if (i->CLRToken)
       OS << binary_le(i->CLRToken->AuxType)
          << zeros(i->CLRToken->unused1)