[ELF] - Implemented threaded --build-id computation

Patch switches computing of --build-id hash to tree.

This is the way when input data is splitted by chunks,
hash is computed for each one in threaded/non-threaded way.
At the end hash is conputed for result tree.

With or without -threads the result hash is the same.

Differential revision: https://reviews.llvm.org/D26199

llvm-svn: 286061

lld/ELF/SyntheticSections.cpp

@@ -23,6 +23,7 @@
 #include "Strings.h"
 #include "SymbolTable.h"
 
+#include "lld/Core/Parallel.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/MD5.h"
 #include "llvm/Support/RandomNumberGenerator.h"
@@ -91,7 +92,8 @@ InterpSection<ELFT>::InterpSection()
 template <class ELFT>
 BuildIdSection<ELFT>::BuildIdSection(size_t HashSize)
     : InputSection<ELFT>(SHF_ALLOC, SHT_NOTE, 1, ArrayRef<uint8_t>(),
-                         ".note.gnu.build-id") {
+                         ".note.gnu.build-id"),
+      HashSize(HashSize) {
   this->Live = true;
 
   Buf.resize(16 + HashSize);
@@ -108,29 +110,66 @@ uint8_t *BuildIdSection<ELFT>::getOutputLoc(uint8_t *Start) const {
   return Start + this->OutSec->getFileOffset() + this->OutSecOff;
 }
 
+static std::vector<ArrayRef<uint8_t>> split(ArrayRef<uint8_t> Arr,
+                                            size_t ChunkSize) {
+  std::vector<ArrayRef<uint8_t>> Ret;
+  while (Arr.size() > ChunkSize) {
+    Ret.push_back(Arr.take_front(ChunkSize));
+    Arr = Arr.drop_front(ChunkSize);
+  }
+  if (!Arr.empty())
+    Ret.push_back(Arr);
+  return Ret;
+}
+
+template <class ELFT>
+void BuildIdSection<ELFT>::computeHash(
+    llvm::ArrayRef<uint8_t> Data,
+    std::function<void(ArrayRef<uint8_t> Arr, uint8_t *Hash)> Hash) {
+  std::vector<ArrayRef<uint8_t>> Chunks = split(Data, 1024 * 1024);
+  std::vector<uint8_t> HashList(Chunks.size() * HashSize);
+
+  if (Config->Threads)
+    parallel_for_each(Chunks.begin(), Chunks.end(),
+                      [&](ArrayRef<uint8_t> &Chunk) {
+                        size_t Id = &Chunk - Chunks.data();
+                        Hash(Chunk, HashList.data() + Id * HashSize);
+                      });
+  else
+    std::for_each(Chunks.begin(), Chunks.end(), [&](ArrayRef<uint8_t> &Chunk) {
+      size_t Id = &Chunk - Chunks.data();
+      Hash(Chunk, HashList.data() + Id * HashSize);
+    });
+
+  Hash(HashList, this->getOutputLoc((uint8_t *)Data.begin()) + 16);
+}
+
 template <class ELFT>
 void BuildIdFastHash<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
-  const endianness E = ELFT::TargetEndianness;
-
-  // 64-bit xxhash
-  uint64_t Hash = xxHash64(toStringRef(Buf));
-  write64<E>(this->getOutputLoc(Buf.begin()) + 16, Hash);
+  computeHash(Buf, [](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
+    uint64_t Hash = xxHash64(toStringRef(Arr));
+    write64<ELFT::TargetEndianness>(Dest, Hash);
+  });
 }
 
 template <class ELFT>
 void BuildIdMd5<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
-  MD5 Hash;
-  Hash.update(Buf);
-  MD5::MD5Result Res;
-  Hash.final(Res);
-  memcpy(this->getOutputLoc(Buf.begin()) + 16, Res, 16);
+  computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
+    MD5 Hash;
+    Hash.update(Arr);
+    MD5::MD5Result Res;
+    Hash.final(Res);
+    memcpy(Dest, Res, HashSize);
+  });
 }
 
 template <class ELFT>
 void BuildIdSha1<ELFT>::writeBuildId(MutableArrayRef<uint8_t> Buf) {
-  SHA1 Hash;
-  Hash.update(Buf);
-  memcpy(this->getOutputLoc(Buf.begin()) + 16, Hash.final().data(), 20);
+  computeHash(Buf, [&](ArrayRef<uint8_t> Arr, uint8_t *Dest) {
+    SHA1 Hash;
+    Hash.update(Arr);
+    memcpy(Dest, Hash.final().data(), HashSize);
+  });
 }
 
 template <class ELFT>

lld/ELF/SyntheticSections.h

@@ -38,6 +38,12 @@ public:
 protected:
   BuildIdSection(size_t HashSize);
   std::vector<uint8_t> Buf;
+
+  void
+  computeHash(llvm::ArrayRef<uint8_t> Buf,
+              std::function<void(ArrayRef<uint8_t> Arr, uint8_t *Hash)> Hash);
+
+  size_t HashSize;
 };
 
 template <class ELFT>

lld/test/ELF/build-id.s

@@ -16,6 +16,14 @@
 # RUN: ld.lld --build-id=md5 --build-id=none %t -o %t2
 # RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=NONE %s
 
+## Multithreaded cases:
+# RUN: ld.lld --build-id -threads %t -o %t2
+# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=DEFAULT %s
+# RUN: ld.lld --build-id=md5 -threads %t -o %t2
+# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=MD5 %s
+# RUN: ld.lld --build-id=sha1 -threads %t -o %t2
+# RUN: llvm-objdump -s %t2 | FileCheck -check-prefix=SHA1 %s
+
 .globl _start
 _start:
   nop
@@ -26,12 +34,19 @@ _start:
 # DEFAULT:      Contents of section .note.test:
 # DEFAULT:      Contents of section .note.gnu.build-id:
 # DEFAULT-NEXT: 04000000 08000000 03000000 474e5500  ............GNU.
+# DEFAULT-NEXT: ab
 
 # MD5:      Contents of section .note.gnu.build-id:
 # MD5-NEXT: 04000000 10000000 03000000 474e5500  ............GNU.
+# MD5-NEXT: 29
 
 # SHA1:      Contents of section .note.gnu.build-id:
 # SHA1-NEXT: 04000000 14000000 03000000 474e5500  ............GNU.
+# SHA1-NEXT: b1
+
+# TREE:      Contents of section .note.gnu.build-id:
+# TREE-NEXT: 04000000 14000000 03000000 474e5500  ............GNU.
+# TREE-NEXT: 18
 
 # UUID:      Contents of section .note.gnu.build-id:
 # UUID-NEXT: 04000000 10000000 03000000 474e5500  ............GNU.