[llvm-objcopy] Implement the PE-COFF specific --subsystem option

This implements the parsing of the highly PE-COFF specific option
in ConfigManager.cpp, setting Optional<> values in COFFConfig, which
then are used in COFFObjcopy.

This should fix https://github.com/mstorsjo/llvm-mingw/issues/239.

Differential Revision: https://reviews.llvm.org/D116556

llvm/docs/CommandGuide/llvm-objcopy.rst

@@ -477,6 +477,13 @@ MACH-O-SPECIFIC OPTIONS
 
  Keep undefined symbols, even if they would otherwise be stripped.
 
+COFF-SPECIFIC OPTIONS
+---------------------
+
+.. option:: --subsystem <name>[:<version>]
+
+ Set the PE subsystem, and optionally subsystem version.
+
 SUPPORTED FORMATS
 -----------------
 

llvm/test/tools/llvm-objcopy/COFF/pe-fields.test

@@ -0,0 +1,20 @@
+## Test that options for altering PE header fields error out on object files.
+
+# RUN: yaml2obj %s -o %t.in.obj
+
+# RUN: not llvm-objcopy --subsystem windows %t.in.obj %t.out.obj 2>&1 | FileCheck %s -DFILE=%t.out.obj
+
+# CHECK: '[[FILE]]': unable to set subsystem on a relocatable object file
+
+--- !COFF
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:
+  - Name:            .text
+    Characteristics: [  ]
+    VirtualAddress:  4096
+    VirtualSize:     1
+    SectionData:     C3
+symbols:
+...

llvm/test/tools/llvm-objcopy/COFF/subsystem.test

@@ -0,0 +1,61 @@
+## Test the --subsystem option.
+
+# RUN: yaml2obj %s -o %t.in.exe
+
+# RUN: llvm-objcopy --subsystem=posix:7.4 --subsystem windows %t.in.exe %t.out.exe
+# RUN: llvm-readobj --file-headers %t.out.exe | FileCheck %s --check-prefix=WIN74
+
+# WIN74: MajorOperatingSystemVersion: 6
+# WIN74: MinorOperatingSystemVersion: 0
+# WIN74: MajorSubsystemVersion: 7
+# WIN74: MinorSubsystemVersion: 4
+# WIN74: Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+
+# RUN: llvm-objcopy --subsystem=posix:7.4 --subsystem windows:9 %t.in.exe %t.out.exe
+# RUN: llvm-readobj --file-headers %t.out.exe | FileCheck %s --check-prefix=WIN90
+
+# WIN90: MajorOperatingSystemVersion: 6
+# WIN90: MinorOperatingSystemVersion: 0
+# WIN90: MajorSubsystemVersion: 9
+# WIN90: MinorSubsystemVersion: 0
+# WIN90: Subsystem: IMAGE_SUBSYSTEM_WINDOWS_GUI
+
+# RUN: not llvm-objcopy --subsystem=foobar %t.in.exe %t.err.exe 2>&1 | FileCheck %s --check-prefix=INVALID-SUBSYS
+
+# INVALID-SUBSYS: 'foobar' is not a valid subsystem{{$}}
+
+# RUN: not llvm-objcopy --subsystem=windows:foo %t.in.exe %t.err.exe 2>&1 | FileCheck %s --check-prefix=INVALID-MAJOR-NUMBER
+# RUN: not llvm-objcopy --subsystem=windows:8.bar %t.in.exe %t.err.exe 2>&1 | FileCheck %s --check-prefix=INVALID-MINOR-NUMBER
+
+# INVALID-MAJOR-NUMBER: 'foo' is not a valid subsystem major version
+# INVALID-MINOR-NUMBER: 'bar' is not a valid subsystem minor version
+
+--- !COFF
+OptionalHeader:
+  AddressOfEntryPoint: 4096
+  ImageBase:       1073741824
+  SectionAlignment: 4096
+  FileAlignment:   512
+  MajorOperatingSystemVersion: 6
+  MinorOperatingSystemVersion: 0
+  MajorImageVersion: 0
+  MinorImageVersion: 0
+  MajorSubsystemVersion: 6
+  MinorSubsystemVersion: 0
+  Subsystem:       IMAGE_SUBSYSTEM_WINDOWS_CUI
+  DLLCharacteristics: [  ]
+  SizeOfStackReserve: 1048576
+  SizeOfStackCommit: 4096
+  SizeOfHeapReserve: 1048576
+  SizeOfHeapCommit: 4096
+header:
+  Machine:         IMAGE_FILE_MACHINE_AMD64
+  Characteristics: [  ]
+sections:
+  - Name:            .text
+    Characteristics: [  ]
+    VirtualAddress:  4096
+    VirtualSize:     1
+    SectionData:     C3
+symbols:
+...

llvm/tools/llvm-objcopy/COFF/COFFConfig.h

@@ -9,11 +9,17 @@
 #ifndef LLVM_TOOLS_LLVM_OBJCOPY_COFF_COFFCONFIG_H
 #define LLVM_TOOLS_LLVM_OBJCOPY_COFF_COFFCONFIG_H
 
+#include "llvm/ADT/Optional.h"
+
 namespace llvm {
 namespace objcopy {
 
 // Coff specific configuration for copying/stripping a single file.
-struct COFFConfig {};
+struct COFFConfig {
+  Optional<unsigned> Subsystem;
+  Optional<unsigned> MajorSubsystemVersion;
+  Optional<unsigned> MinorSubsystemVersion;
+};
 
 } // namespace objcopy
 } // namespace llvm

llvm/tools/llvm-objcopy/COFF/COFFObjcopy.cpp

@@ -130,7 +130,8 @@ static uint32_t flagsToCharacteristics(SectionFlag AllFlags, uint32_t OldChar) {
   return NewCharacteristics;
 }
 
-static Error handleArgs(const CommonConfig &Config, Object &Obj) {
+static Error handleArgs(const CommonConfig &Config,
+                        const COFFConfig &COFFConfig, Object &Obj) {
   // Perform the actual section removals.
   Obj.removeSections([&Config](const Section &Sec) {
     // Contrary to --only-keep-debug, --only-section fully removes sections that
@@ -256,18 +257,34 @@ static Error handleArgs(const CommonConfig &Config, Object &Obj) {
     if (Error E = addGnuDebugLink(Obj, Config.AddGnuDebugLink))
       return E;
 
+  if (COFFConfig.Subsystem || COFFConfig.MajorSubsystemVersion ||
+      COFFConfig.MinorSubsystemVersion) {
+    if (!Obj.IsPE)
+      return createStringError(
+          errc::invalid_argument,
+          "'" + Config.OutputFilename +
+              "': unable to set subsystem on a relocatable object file");
+    if (COFFConfig.Subsystem)
+      Obj.PeHeader.Subsystem = *COFFConfig.Subsystem;
+    if (COFFConfig.MajorSubsystemVersion)
+      Obj.PeHeader.MajorSubsystemVersion = *COFFConfig.MajorSubsystemVersion;
+    if (COFFConfig.MinorSubsystemVersion)
+      Obj.PeHeader.MinorSubsystemVersion = *COFFConfig.MinorSubsystemVersion;
+  }
+
   return Error::success();
 }
 
-Error executeObjcopyOnBinary(const CommonConfig &Config, const COFFConfig &,
-                             COFFObjectFile &In, raw_ostream &Out) {
+Error executeObjcopyOnBinary(const CommonConfig &Config,
+                             const COFFConfig &COFFConfig, COFFObjectFile &In,
+                             raw_ostream &Out) {
   COFFReader Reader(In);
   Expected<std::unique_ptr<Object>> ObjOrErr = Reader.create();
   if (!ObjOrErr)
     return createFileError(Config.InputFilename, ObjOrErr.takeError());
   Object *Obj = ObjOrErr->get();
   assert(Obj && "Unable to deserialize COFF object");
-  if (Error E = handleArgs(Config, *Obj))
+  if (Error E = handleArgs(Config, COFFConfig, *Obj))
     return createFileError(Config.InputFilename, std::move(E));
   COFFWriter Writer(*Obj, Out);
   if (Error E = Writer.write())

llvm/tools/llvm-objcopy/ConfigManager.cpp

@@ -11,6 +11,7 @@
 #include "llvm/ADT/SmallVector.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/StringSet.h"
+#include "llvm/BinaryFormat/COFF.h"
 #include "llvm/Option/Arg.h"
 #include "llvm/Option/ArgList.h"
 #include "llvm/Support/CRC.h"
@@ -675,6 +676,7 @@ objcopy::parseObjcopyOptions(ArrayRef<const char *> RawArgsArr,
 
   ConfigManager ConfigMgr;
   CommonConfig &Config = ConfigMgr.Common;
+  COFFConfig &COFFConfig = ConfigMgr.COFF;
   ELFConfig &ELFConfig = ConfigMgr.ELF;
   MachOConfig &MachOConfig = ConfigMgr.MachO;
   Config.InputFilename = Positional[0];
@@ -733,6 +735,46 @@ objcopy::parseObjcopyOptions(ArrayRef<const char *> RawArgsArr,
                                VisibilityStr.str().c_str());
   }
 
+  for (const auto *Arg : InputArgs.filtered(OBJCOPY_subsystem)) {
+    StringRef Subsystem, Version;
+    std::tie(Subsystem, Version) = StringRef(Arg->getValue()).split(':');
+    COFFConfig.Subsystem =
+        StringSwitch<unsigned>(Subsystem.lower())
+            .Case("boot_application",
+                  COFF::IMAGE_SUBSYSTEM_WINDOWS_BOOT_APPLICATION)
+            .Case("console", COFF::IMAGE_SUBSYSTEM_WINDOWS_CUI)
+            .Case("efi_application", COFF::IMAGE_SUBSYSTEM_EFI_APPLICATION)
+            .Case("efi_boot_service_driver",
+                  COFF::IMAGE_SUBSYSTEM_EFI_BOOT_SERVICE_DRIVER)
+            .Case("efi_rom", COFF::IMAGE_SUBSYSTEM_EFI_ROM)
+            .Case("efi_runtime_driver",
+                  COFF::IMAGE_SUBSYSTEM_EFI_RUNTIME_DRIVER)
+            .Case("native", COFF::IMAGE_SUBSYSTEM_NATIVE)
+            .Case("posix", COFF::IMAGE_SUBSYSTEM_POSIX_CUI)
+            .Case("windows", COFF::IMAGE_SUBSYSTEM_WINDOWS_GUI)
+            .Default(COFF::IMAGE_SUBSYSTEM_UNKNOWN);
+    if (*COFFConfig.Subsystem == COFF::IMAGE_SUBSYSTEM_UNKNOWN)
+      return createStringError(errc::invalid_argument,
+                               "'%s' is not a valid subsystem",
+                               Subsystem.str().c_str());
+    if (!Version.empty()) {
+      StringRef Major, Minor;
+      std::tie(Major, Minor) = Version.split('.');
+      unsigned Number;
+      if (Major.getAsInteger(10, Number))
+        return createStringError(errc::invalid_argument,
+                                 "'%s' is not a valid subsystem major version",
+                                 Major.str().c_str());
+      COFFConfig.MajorSubsystemVersion = Number;
+      Number = 0;
+      if (!Minor.empty() && Minor.getAsInteger(10, Number))
+        return createStringError(errc::invalid_argument,
+                                 "'%s' is not a valid subsystem minor version",
+                                 Minor.str().c_str());
+      COFFConfig.MinorSubsystemVersion = Number;
+    }
+  }
+
   Config.OutputFormat = StringSwitch<FileFormat>(OutputFormat)
                             .Case("binary", FileFormat::Binary)
                             .Case("ihex", FileFormat::IHex)

llvm/tools/llvm-objcopy/ObjcopyOpts.td

@@ -105,6 +105,11 @@ defm strip_unneeded_symbols
          "if they are not needed by relocations">,
       MetaVarName<"filename">;
 
+defm subsystem
+    : Eq<"subsystem",
+         "Set PE subsystem and version">,
+      MetaVarName<"name[:version]">;
+
 def extract_dwo
     : Flag<["--"], "extract-dwo">,
       HelpText<