diff --git a/lld/ELF/ICF.cpp b/lld/ELF/ICF.cpp
index a3e098082f30..5f6b41ef739d 100644
--- a/lld/ELF/ICF.cpp
+++ b/lld/ELF/ICF.cpp
@@ -120,10 +120,7 @@ private:
 // relocation targets is not included in the hash value.
 template <class ELFT> uint64_t ICF<ELFT>::getHash(InputSection<ELFT> *S) {
   uint64_t Flags = S->Flags;
-  uint64_t H = hash_combine(Flags, S->getSize());
-  for (const Elf_Shdr *Rel : S->RelocSections)
-    H = hash_combine(H, (uint64_t)Rel->sh_size);
-  return H;
+  return hash_combine(Flags, S->getSize(), S->NumRelocations);
 }
 
 // Returns true if Sec is subject of ICF.
@@ -212,21 +209,15 @@ bool ICF<ELFT>::relocationEq(ArrayRef<RelTy> RelsA, ArrayRef<RelTy> RelsB) {
 template <class ELFT>
 bool ICF<ELFT>::equalsConstant(const InputSection<ELFT> *A,
                                const InputSection<ELFT> *B) {
-  if (A->RelocSections.size() != B->RelocSections.size())
+  if (A->NumRelocations != B->NumRelocations)
     return false;
 
-  for (size_t I = 0, E = A->RelocSections.size(); I != E; ++I) {
-    const Elf_Shdr *RA = A->RelocSections[I];
-    const Elf_Shdr *RB = B->RelocSections[I];
-    ELFFile<ELFT> FileA = A->getObj();
-    ELFFile<ELFT> FileB = B->getObj();
-    if (RA->sh_type == SHT_RELA) {
-      if (!relocationEq(check(FileA.relas(RA)), check(FileB.relas(RB))))
-        return false;
-    } else {
-      if (!relocationEq(check(FileA.rels(RA)), check(FileB.rels(RB))))
-        return false;
-    }
+  if (A->AreRelocsRela) {
+    if (!relocationEq(A->relas(), B->relas()))
+      return false;
+  } else {
+    if (!relocationEq(A->rels(), B->rels()))
+      return false;
   }
 
   return A->Flags == B->Flags && A->getSize() == B->getSize() &&
@@ -268,20 +259,9 @@ bool ICF<ELFT>::variableEq(const InputSection<ELFT> *A,
 template <class ELFT>
 bool ICF<ELFT>::equalsVariable(const InputSection<ELFT> *A,
                                const InputSection<ELFT> *B) {
-  for (size_t I = 0, E = A->RelocSections.size(); I != E; ++I) {
-    const Elf_Shdr *RA = A->RelocSections[I];
-    const Elf_Shdr *RB = B->RelocSections[I];
-    ELFFile<ELFT> FileA = A->getObj();
-    ELFFile<ELFT> FileB = B->getObj();
-    if (RA->sh_type == SHT_RELA) {
-      if (!variableEq(A, B, check(FileA.relas(RA)), check(FileB.relas(RB))))
-        return false;
-    } else {
-      if (!variableEq(A, B, check(FileA.rels(RA)), check(FileB.rels(RB))))
-        return false;
-    }
-  }
-  return true;
+  if (A->AreRelocsRela)
+    return variableEq(A, B, A->relas(), B->relas());
+  return variableEq(A, B, A->rels(), B->rels());
 }
 
 // The main function of ICF.
diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp
index 340c98e57923..3eb89097d694 100644
--- a/lld/ELF/InputFiles.cpp
+++ b/lld/ELF/InputFiles.cpp
@@ -353,19 +353,28 @@ elf::ObjectFile<ELFT>::createInputSection(const Elf_Shdr &Sec,
     InputSectionBase<ELFT> *Target = getRelocTarget(Sec);
     if (!Target)
       return nullptr;
-    if (auto *S = dyn_cast<InputSection<ELFT>>(Target)) {
-      S->RelocSections.push_back(&Sec);
-      return nullptr;
+    if (Target->FirstRelocation)
+      fatal(getFilename(this) +
+            ": multiple relocation sections to one section are not supported");
+    if (!isa<InputSection<ELFT>>(Target) && !isa<EhInputSection<ELFT>>(Target))
+      fatal(getFilename(this) +
+            ": relocations pointing to SHF_MERGE are not supported");
+
+    size_t NumRelocations;
+    if (Sec.sh_type == SHT_RELA) {
+      ArrayRef<Elf_Rela> Rels = check(this->getObj().relas(&Sec));
+      Target->FirstRelocation = Rels.begin();
+      NumRelocations = Rels.size();
+      Target->AreRelocsRela = true;
+    } else {
+      ArrayRef<Elf_Rel> Rels = check(this->getObj().rels(&Sec));
+      Target->FirstRelocation = Rels.begin();
+      NumRelocations = Rels.size();
+      Target->AreRelocsRela = false;
     }
-    if (auto *S = dyn_cast<EhInputSection<ELFT>>(Target)) {
-      if (S->RelocSection)
-        fatal(getFilename(this) +
-              ": multiple relocation sections to .eh_frame are not supported");
-      S->RelocSection = &Sec;
-      return nullptr;
-    }
-    fatal(getFilename(this) +
-          ": relocations pointing to SHF_MERGE are not supported");
+    assert(isUInt<31>(NumRelocations));
+    Target->NumRelocations = NumRelocations;
+    return nullptr;
   }
   }
 
diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h
index e867acf5bfd9..6115bb2d8c40 100644
--- a/lld/ELF/InputFiles.h
+++ b/lld/ELF/InputFiles.h
@@ -121,6 +121,8 @@ protected:
 // .o file.
 template <class ELFT> class ObjectFile : public ELFFileBase<ELFT> {
   typedef ELFFileBase<ELFT> Base;
+  typedef typename ELFT::Rel Elf_Rel;
+  typedef typename ELFT::Rela Elf_Rela;
   typedef typename ELFT::Sym Elf_Sym;
   typedef typename ELFT::Shdr Elf_Shdr;
   typedef typename ELFT::SymRange Elf_Sym_Range;
diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp
index d4d854e9874b..83fb3aa601b7 100644
--- a/lld/ELF/InputSection.cpp
+++ b/lld/ELF/InputSection.cpp
@@ -56,6 +56,9 @@ InputSectionBase<ELFT>::InputSectionBase(elf::ObjectFile<ELFT> *File,
                        !Config->GcSections || !(Flags & SHF_ALLOC)),
       File(File), Flags(Flags), Entsize(Entsize), Type(Type), Link(Link),
       Info(Info), Repl(this) {
+  NumRelocations = 0;
+  AreRelocsRela = false;
+
   // The ELF spec states that a value of 0 means the section has
   // no alignment constraits.
   uint64_t V = std::max<uint64_t>(Addralign, 1);
@@ -466,12 +469,10 @@ void InputSectionBase<ELFT>::relocate(uint8_t *Buf, uint8_t *BufEnd) {
   // we handle relocations directly here.
   auto *IS = dyn_cast<InputSection<ELFT>>(this);
   if (IS && !(IS->Flags & SHF_ALLOC)) {
-    for (const Elf_Shdr *RelSec : IS->RelocSections) {
-      if (RelSec->sh_type == SHT_RELA)
-        IS->relocateNonAlloc(Buf, check(IS->getObj().relas(RelSec)));
-      else
-        IS->relocateNonAlloc(Buf, check(IS->getObj().rels(RelSec)));
-    }
+    if (IS->AreRelocsRela)
+      IS->relocateNonAlloc(Buf, IS->relas());
+    else
+      IS->relocateNonAlloc(Buf, IS->rels());
     return;
   }
 
@@ -609,12 +610,11 @@ template <class ELFT> void EhInputSection<ELFT>::split() {
   if (!this->Pieces.empty())
     return;
 
-  if (RelocSection) {
-    ELFFile<ELFT> Obj = this->getObj();
-    if (RelocSection->sh_type == SHT_RELA)
-      split(check(Obj.relas(RelocSection)));
+  if (this->NumRelocations) {
+    if (this->AreRelocsRela)
+      split(this->relas());
     else
-      split(check(Obj.rels(RelocSection)));
+      split(this->rels());
     return;
   }
   split(makeArrayRef<typename ELFT::Rela>(nullptr, nullptr));
diff --git a/lld/ELF/InputSection.h b/lld/ELF/InputSection.h
index d6032af3a8b9..d39ee4c6189f 100644
--- a/lld/ELF/InputSection.h
+++ b/lld/ELF/InputSection.h
@@ -96,7 +96,10 @@ public:
 
   InputSectionBase()
       : InputSectionData(Regular, "", ArrayRef<uint8_t>(), false, false),
-        Repl(this) {}
+        Repl(this) {
+    NumRelocations = 0;
+    AreRelocsRela = false;
+  }
 
   InputSectionBase(ObjectFile<ELFT> *File, const Elf_Shdr *Header,
                    StringRef Name, Kind SectionKind);
@@ -106,6 +109,20 @@ public:
                    Kind SectionKind);
   OutputSectionBase *OutSec = nullptr;
 
+  // Relocations that refer to this section.
+  const Elf_Rel *FirstRelocation = nullptr;
+  unsigned NumRelocations : 31;
+  unsigned AreRelocsRela : 1;
+  ArrayRef<Elf_Rel> rels() const {
+    assert(!AreRelocsRela);
+    return llvm::makeArrayRef(FirstRelocation, NumRelocations);
+  }
+  ArrayRef<Elf_Rela> relas() const {
+    assert(AreRelocsRela);
+    return llvm::makeArrayRef(static_cast<const Elf_Rela *>(FirstRelocation),
+                              NumRelocations);
+  }
+
   // This pointer points to the "real" instance of this instance.
   // Usually Repl == this. However, if ICF merges two sections,
   // Repl pointer of one section points to another section. So,
@@ -219,8 +236,6 @@ public:
   // rather than a single large blob of data.
   std::vector<EhSectionPiece> Pieces;
 
-  // Relocation section that refer to this one.
-  const Elf_Shdr *RelocSection = nullptr;
 };
 
 // This corresponds to a non SHF_MERGE section of an input file.
@@ -247,9 +262,6 @@ public:
   // beginning of the output section.
   void writeTo(uint8_t *Buf);
 
-  // Relocation sections that refer to this one.
-  llvm::TinyPtrVector<const Elf_Shdr *> RelocSections;
-
   // The offset from beginning of the output sections this section was assigned
   // to. The writer sets a value.
   uint64_t OutSecOff = 0;
diff --git a/lld/ELF/MarkLive.cpp b/lld/ELF/MarkLive.cpp
index 60995a703338..8d129fc3ff13 100644
--- a/lld/ELF/MarkLive.cpp
+++ b/lld/ELF/MarkLive.cpp
@@ -80,15 +80,12 @@ static ResolvedReloc<ELFT> resolveReloc(InputSectionBase<ELFT> &Sec,
 template <class ELFT>
 static void forEachSuccessor(InputSection<ELFT> &Sec,
                              std::function<void(ResolvedReloc<ELFT>)> Fn) {
-  ELFFile<ELFT> Obj = Sec.getFile()->getObj();
-  for (const typename ELFT::Shdr *RelSec : Sec.RelocSections) {
-    if (RelSec->sh_type == SHT_RELA) {
-      for (const typename ELFT::Rela &Rel : check(Obj.relas(RelSec)))
-        Fn(resolveReloc(Sec, Rel));
-    } else {
-      for (const typename ELFT::Rel &Rel : check(Obj.rels(RelSec)))
-        Fn(resolveReloc(Sec, Rel));
-    }
+  if (Sec.AreRelocsRela) {
+    for (const typename ELFT::Rela &Rel : Sec.relas())
+      Fn(resolveReloc(Sec, Rel));
+  } else {
+    for (const typename ELFT::Rel &Rel : Sec.rels())
+      Fn(resolveReloc(Sec, Rel));
   }
   if (Sec.DependentSection)
     Fn({Sec.DependentSection, 0});
@@ -146,18 +143,17 @@ template <class ELFT>
 static void
 scanEhFrameSection(EhInputSection<ELFT> &EH,
                    std::function<void(ResolvedReloc<ELFT>)> Enqueue) {
-  if (!EH.RelocSection)
+  if (!EH.NumRelocations)
     return;
 
   // Unfortunately we need to split .eh_frame early since some relocations in
   // .eh_frame keep other section alive and some don't.
   EH.split();
 
-  ELFFile<ELFT> EObj = EH.getFile()->getObj();
-  if (EH.RelocSection->sh_type == SHT_RELA)
-    scanEhFrameSection(EH, check(EObj.relas(EH.RelocSection)), Enqueue);
+  if (EH.AreRelocsRela)
+    scanEhFrameSection(EH, EH.relas(), Enqueue);
   else
-    scanEhFrameSection(EH, check(EObj.rels(EH.RelocSection)), Enqueue);
+    scanEhFrameSection(EH, EH.rels(), Enqueue);
 }
 
 // We do not garbage-collect two types of sections:
diff --git a/lld/ELF/OutputSections.cpp b/lld/ELF/OutputSections.cpp
index 5402c3a40a5b..c2c37f20f75a 100644
--- a/lld/ELF/OutputSections.cpp
+++ b/lld/ELF/OutputSections.cpp
@@ -1162,12 +1162,11 @@ void EhOutputSection<ELFT>::addSection(InputSectionData *C) {
   if (Sec->Pieces.empty())
     return;
 
-  if (const Elf_Shdr *RelSec = Sec->RelocSection) {
-    ELFFile<ELFT> Obj = Sec->getFile()->getObj();
-    if (RelSec->sh_type == SHT_RELA)
-      addSectionAux(Sec, check(Obj.relas(RelSec)));
+  if (Sec->NumRelocations) {
+    if (Sec->AreRelocsRela)
+      addSectionAux(Sec, Sec->relas());
     else
-      addSectionAux(Sec, check(Obj.rels(RelSec)));
+      addSectionAux(Sec, Sec->rels());
     return;
   }
   addSectionAux(Sec, makeArrayRef<Elf_Rela>(nullptr, nullptr));
diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp
index 190e303f31cb..07ba93022b60 100644
--- a/lld/ELF/Relocations.cpp
+++ b/lld/ELF/Relocations.cpp
@@ -787,14 +787,11 @@ static void scanRelocs(InputSectionBase<ELFT> &C, ArrayRef<RelTy> Rels) {
   }
 }
 
-template <class ELFT>
-void scanRelocations(InputSectionBase<ELFT> &S,
-                     const typename ELFT::Shdr &RelSec) {
-  ELFFile<ELFT> EObj = S.getFile()->getObj();
-  if (RelSec.sh_type == SHT_RELA)
-    scanRelocs(S, check(EObj.relas(&RelSec)));
+template <class ELFT> void scanRelocations(InputSectionBase<ELFT> &S) {
+  if (S.AreRelocsRela)
+    scanRelocs(S, S.relas());
   else
-    scanRelocs(S, check(EObj.rels(&RelSec)));
+    scanRelocs(S, S.rels());
 }
 
 template <class ELFT, class RelTy>
@@ -817,33 +814,22 @@ static void createThunks(InputSectionBase<ELFT> &C, ArrayRef<RelTy> Rels) {
   }
 }
 
-template <class ELFT>
-void createThunks(InputSectionBase<ELFT> &S,
-                  const typename ELFT::Shdr &RelSec) {
-  ELFFile<ELFT> EObj = S.getFile()->getObj();
-  if (RelSec.sh_type == SHT_RELA)
-    createThunks(S, check(EObj.relas(&RelSec)));
+template <class ELFT> void createThunks(InputSectionBase<ELFT> &S) {
+  if (S.AreRelocsRela)
+    createThunks(S, S.relas());
   else
-    createThunks(S, check(EObj.rels(&RelSec)));
+    createThunks(S, S.rels());
 }
 
-template void scanRelocations<ELF32LE>(InputSectionBase<ELF32LE> &,
-                                       const ELF32LE::Shdr &);
-template void scanRelocations<ELF32BE>(InputSectionBase<ELF32BE> &,
-                                       const ELF32BE::Shdr &);
-template void scanRelocations<ELF64LE>(InputSectionBase<ELF64LE> &,
-                                       const ELF64LE::Shdr &);
-template void scanRelocations<ELF64BE>(InputSectionBase<ELF64BE> &,
-                                       const ELF64BE::Shdr &);
+template void scanRelocations<ELF32LE>(InputSectionBase<ELF32LE> &);
+template void scanRelocations<ELF32BE>(InputSectionBase<ELF32BE> &);
+template void scanRelocations<ELF64LE>(InputSectionBase<ELF64LE> &);
+template void scanRelocations<ELF64BE>(InputSectionBase<ELF64BE> &);
 
-template void createThunks<ELF32LE>(InputSectionBase<ELF32LE> &,
-                                    const ELF32LE::Shdr &);
-template void createThunks<ELF32BE>(InputSectionBase<ELF32BE> &,
-                                    const ELF32BE::Shdr &);
-template void createThunks<ELF64LE>(InputSectionBase<ELF64LE> &,
-                                    const ELF64LE::Shdr &);
-template void createThunks<ELF64BE>(InputSectionBase<ELF64BE> &,
-                                    const ELF64BE::Shdr &);
+template void createThunks<ELF32LE>(InputSectionBase<ELF32LE> &);
+template void createThunks<ELF32BE>(InputSectionBase<ELF32BE> &);
+template void createThunks<ELF64LE>(InputSectionBase<ELF64LE> &);
+template void createThunks<ELF64BE>(InputSectionBase<ELF64BE> &);
 
 template std::string getLocation<ELF32LE>(InputSectionBase<ELF32LE> &S,
                                           uint32_t Offset);
diff --git a/lld/ELF/Relocations.h b/lld/ELF/Relocations.h
index f4b771b8bda8..d14b0a7ad952 100644
--- a/lld/ELF/Relocations.h
+++ b/lld/ELF/Relocations.h
@@ -81,11 +81,9 @@ struct Relocation {
   SymbolBody *Sym;
 };
 
-template <class ELFT>
-void scanRelocations(InputSectionBase<ELFT> &, const typename ELFT::Shdr &);
+template <class ELFT> void scanRelocations(InputSectionBase<ELFT> &);
 
-template <class ELFT>
-void createThunks(InputSectionBase<ELFT> &, const typename ELFT::Shdr &);
+template <class ELFT> void createThunks(InputSectionBase<ELFT> &);
 
 template <class ELFT>
 std::string getLocation(InputSectionBase<ELFT> &S, typename ELFT::uint Offset);
diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp
index 5a45a4a35a8a..3a6ead85d5ce 100644
--- a/lld/ELF/Writer.cpp
+++ b/lld/ELF/Writer.cpp
@@ -54,8 +54,7 @@ private:
   void addReservedSymbols();
   void addInputSec(InputSectionBase<ELFT> *S);
   void createSections();
-  void forEachRelSec(std::function<void(InputSectionBase<ELFT> &,
-                                        const typename ELFT::Shdr &)> Fn);
+  void forEachRelSec(std::function<void(InputSectionBase<ELFT> &)> Fn);
   void sortSections();
   void finalizeSections();
   void addPredefinedSections();
@@ -694,8 +693,7 @@ static void sortBySymbolsOrder(ArrayRef<OutputSectionBase *> V) {
 
 template <class ELFT>
 void Writer<ELFT>::forEachRelSec(
-    std::function<void(InputSectionBase<ELFT> &, const typename ELFT::Shdr &)>
-        Fn) {
+    std::function<void(InputSectionBase<ELFT> &)> Fn) {
   for (InputSectionBase<ELFT> *IS : Symtab<ELFT>::X->Sections) {
     if (!IS->Live)
       continue;
@@ -706,14 +704,8 @@ void Writer<ELFT>::forEachRelSec(
     // processed by InputSection::relocateNonAlloc.
     if (!(IS->Flags & SHF_ALLOC))
       continue;
-    if (auto *S = dyn_cast<InputSection<ELFT>>(IS)) {
-      for (const Elf_Shdr *RelSec : S->RelocSections)
-        Fn(*S, *RelSec);
-      continue;
-    }
-    if (auto *S = dyn_cast<EhInputSection<ELFT>>(IS))
-      if (S->RelocSection)
-        Fn(*S, *S->RelocSection);
+    if (isa<InputSection<ELFT>>(IS) || isa<EhInputSection<ELFT>>(IS))
+      Fn(*IS);
   }
 }
 
diff --git a/lld/test/ELF/invalid/invalid-elf.test b/lld/test/ELF/invalid/invalid-elf.test
index a20c4b4e7449..d49ab2f3cacf 100644
--- a/lld/test/ELF/invalid/invalid-elf.test
+++ b/lld/test/ELF/invalid/invalid-elf.test
@@ -26,6 +26,6 @@
 
 # RUN: not ld.lld %p/Inputs/multiple-eh-relocs.elf -o %t2 2>&1 | \
 # RUN:   FileCheck --check-prefix=INVALID-EH-RELOCS %s
-# INVALID-EH-RELOCS: multiple relocation sections to .eh_frame are not supported
+# INVALID-EH-RELOCS: multiple relocation sections to one section are not supported
 
 .long foo