Add some boilerplate to the PTH file to prepare for the caching of stats for directories (and negative stats too).

llvm-svn: 64477

clang/Driver/CacheTokens.cpp

@@ -1,4 +1,4 @@
-//===--- CacheTokens.cpp - Caching of lexer tokens for PCH support --------===//
+//===--- CacheTokens.cpp - Caching of lexer tokens for PTH support --------===//
 //
 //                     The LLVM Compiler Infrastructure
 //
@@ -7,7 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 //
-// This provides a possible implementation of PCH support for Clang that is
+// This provides a possible implementation of PTH support for Clang that is
 // based on caching lexed tokens and identifiers.
 //
 //===----------------------------------------------------------------------===//
@@ -192,54 +192,115 @@ public:
 //===----------------------------------------------------------------------===//
 
 namespace {
-class VISIBILITY_HIDDEN PCHEntry {
+class VISIBILITY_HIDDEN PTHEntry {
   Offset TokenData, PPCondData;  
 
 public:  
-  PCHEntry() {}
+  PTHEntry() {}
 
-  PCHEntry(Offset td, Offset ppcd)
+  PTHEntry(Offset td, Offset ppcd)
     : TokenData(td), PPCondData(ppcd) {}
   
   Offset getTokenOffset() const { return TokenData; }  
   Offset getPPCondTableOffset() const { return PPCondData; }
 };
   
-class VISIBILITY_HIDDEN FileEntryPCHEntryInfo {
+  
+class VISIBILITY_HIDDEN PTHEntryKeyVariant {
+  union { const FileEntry* FE; const DirectoryEntry* DE; const char* Path; };
+  enum { IsFE = 0x1, IsDE = 0x2, IsNoExist = 0x0 } Kind;
 public:
-  typedef const FileEntry* key_type;
+  PTHEntryKeyVariant(const FileEntry *fe) : FE(fe), Kind(IsFE) {}
+  PTHEntryKeyVariant(const DirectoryEntry *de) : DE(de), Kind(IsDE) {}
+  PTHEntryKeyVariant(const char* path) : Path(path), Kind(IsNoExist) {}
+  
+  const FileEntry *getFile() const { return Kind == IsFE ? FE : 0; }
+  const DirectoryEntry *getDir() const { return Kind == IsDE ? DE : 0; }
+  const char* getNameOfNonExistantFile() const {
+    return Kind == IsNoExist ? Path : 0;
+  }
+  
+  const char* getCString() const {
+    switch (Kind) {
+      case IsFE: return FE->getName();
+      case IsDE: return DE->getName();
+      default: return Path;
+    }
+  }
+  
+  unsigned getKind() const { return (unsigned) Kind; }
+  
+  void EmitData(llvm::raw_ostream& Out) {
+    switch (Kind) {
+      case IsFE:
+        // Emit stat information.
+        ::Emit32(Out, FE->getInode());
+        ::Emit32(Out, FE->getDevice());
+        ::Emit16(Out, FE->getFileMode());
+        ::Emit64(Out, FE->getModificationTime());
+        ::Emit64(Out, FE->getSize());
+        break;
+      case IsDE:
+        // FIXME
+      default: break;
+        // Emit nothing.        
+    }
+  }
+  
+  unsigned getRepresentationLength() const {
+    switch (Kind) {
+      case IsFE: return 4 + 4 + 2 + 8 + 8;
+      case IsDE: // FIXME
+      default: return 0;
+    }
+  }
+};
+  
+class VISIBILITY_HIDDEN FileEntryPTHEntryInfo {
+public:
+  typedef PTHEntryKeyVariant key_type;
   typedef key_type key_type_ref;
   
-  typedef PCHEntry data_type;
+  typedef PTHEntry data_type;
-  typedef const PCHEntry& data_type_ref;
+  typedef const PTHEntry& data_type_ref;
   
-  static unsigned ComputeHash(const FileEntry* FE) {
+  static unsigned ComputeHash(PTHEntryKeyVariant V) {
-    return BernsteinHash(FE->getName());
+    return BernsteinHash(V.getCString());
   }
   
   static std::pair<unsigned,unsigned> 
-  EmitKeyDataLength(llvm::raw_ostream& Out, const FileEntry* FE,
+  EmitKeyDataLength(llvm::raw_ostream& Out, PTHEntryKeyVariant V,
-                    const PCHEntry& E) {
+                    const PTHEntry& E) {
 
-    unsigned n = strlen(FE->getName()) + 1;
+    unsigned n = strlen(V.getCString()) + 1 + 1;
     ::Emit16(Out, n);
-    return std::make_pair(n,(4*2)+(4+4+2+8+8));
+    
+    unsigned m = V.getRepresentationLength() + (V.getFile() ? 4 + 4 : 0);
+    ::Emit8(Out, m);
+
+    return std::make_pair(n, m);
   }
   
-  static void EmitKey(llvm::raw_ostream& Out, const FileEntry* FE, unsigned n) {
+  static void EmitKey(llvm::raw_ostream& Out, PTHEntryKeyVariant V, unsigned n){
-    Out.write(FE->getName(), n);
+    // Emit the entry kind.
+    ::Emit8(Out, (unsigned) V.getKind());
+    // Emit the string.
+    Out.write(V.getCString(), n - 1);
   }
   
-  static void EmitData(llvm::raw_ostream& Out, const FileEntry* FE, 
+  static void EmitData(llvm::raw_ostream& Out, PTHEntryKeyVariant V, 
-                       const PCHEntry& E, unsigned) {
+                       const PTHEntry& E, unsigned) {
-    ::Emit32(Out, E.getTokenOffset());
+
-    ::Emit32(Out, E.getPPCondTableOffset());
+
-    // Emit stat information.
+    // For file entries emit the offsets into the PTH file for token data
-    ::Emit32(Out, FE->getInode());
+    // and the preprocessor blocks table.
-    ::Emit32(Out, FE->getDevice());
+    if (V.getFile()) {
-    ::Emit16(Out, FE->getFileMode());
+      ::Emit32(Out, E.getTokenOffset());
-    ::Emit64(Out, FE->getModificationTime());
+      ::Emit32(Out, E.getPPCondTableOffset());
-    ::Emit64(Out, FE->getSize());
+    }
+    
+    // Emit any other data associated with the key (i.e., stat information).
+    V.EmitData(Out);
   }        
 };
   
@@ -254,7 +315,7 @@ public:
 };
 } // end anonymous namespace
 
-typedef OnDiskChainedHashTableGenerator<FileEntryPCHEntryInfo> PCHMap;
+typedef OnDiskChainedHashTableGenerator<FileEntryPTHEntryInfo> PTHMap;
 typedef llvm::DenseMap<const IdentifierInfo*,uint32_t> IDMap;
 typedef llvm::StringMap<OffsetOpt, llvm::BumpPtrAllocator> CachedStrsTy;
 
@@ -264,7 +325,7 @@ class VISIBILITY_HIDDEN PTHWriter {
   llvm::raw_fd_ostream& Out;
   Preprocessor& PP;
   uint32_t idcount;
-  PCHMap PM;
+  PTHMap PM;
   CachedStrsTy CachedStrs;
   Offset CurStrOffset;
   std::vector<llvm::StringMapEntry<OffsetOpt>*> StrEntries;
@@ -304,7 +365,7 @@ class VISIBILITY_HIDDEN PTHWriter {
   /// token data.
   Offset EmitFileTable() { return PM.Emit(Out); }
 
-  PCHEntry LexTokens(Lexer& L);
+  PTHEntry LexTokens(Lexer& L);
   Offset EmitCachedSpellings();
   
 public:
@@ -360,7 +421,7 @@ void PTHWriter::EmitToken(const Token& T) {
   Emit32(PP.getSourceManager().getFileOffset(T.getLocation()));
 }
 
-PCHEntry PTHWriter::LexTokens(Lexer& L) {
+PTHEntry PTHWriter::LexTokens(Lexer& L) {
   // Pad 0's so that we emit tokens to a 4-byte alignment.
   // This speed up reading them back in.
   Pad(Out, 4);
@@ -513,7 +574,7 @@ PCHEntry PTHWriter::LexTokens(Lexer& L) {
     Emit32(x == i ? 0 : x);
   }
 
-  return PCHEntry(off, PPCondOff);
+  return PTHEntry(off, PPCondOff);
 }
 
 Offset PTHWriter::EmitCachedSpellings() {
@@ -604,39 +665,39 @@ void clang::CacheTokens(Preprocessor& PP, const std::string& OutFile) {
 //===----------------------------------------------------------------------===//
 
 namespace {
-class VISIBILITY_HIDDEN PCHIdKey {
+class VISIBILITY_HIDDEN PTHIdKey {
 public:
   const IdentifierInfo* II;
   uint32_t FileOffset;
 };
 
-class VISIBILITY_HIDDEN PCHIdentifierTableTrait {
+class VISIBILITY_HIDDEN PTHIdentifierTableTrait {
 public:
-  typedef PCHIdKey* key_type;
+  typedef PTHIdKey* key_type;
   typedef key_type  key_type_ref;
   
   typedef uint32_t  data_type;
   typedef data_type data_type_ref;
   
-  static unsigned ComputeHash(PCHIdKey* key) {
+  static unsigned ComputeHash(PTHIdKey* key) {
     return BernsteinHash(key->II->getName());
   }
   
   static std::pair<unsigned,unsigned> 
-  EmitKeyDataLength(llvm::raw_ostream& Out, const PCHIdKey* key, uint32_t) {    
+  EmitKeyDataLength(llvm::raw_ostream& Out, const PTHIdKey* key, uint32_t) {    
     unsigned n = strlen(key->II->getName()) + 1;
     ::Emit16(Out, n);
     return std::make_pair(n, sizeof(uint32_t));
   }
   
-  static void EmitKey(llvm::raw_fd_ostream& Out, PCHIdKey* key, unsigned n) {
+  static void EmitKey(llvm::raw_fd_ostream& Out, PTHIdKey* key, unsigned n) {
     // Record the location of the key data.  This is used when generating
     // the mapping from persistent IDs to strings.
     key->FileOffset = Out.tell();
     Out.write(key->II->getName(), n);
   }
   
-  static void EmitData(llvm::raw_ostream& Out, PCHIdKey*, uint32_t pID,
+  static void EmitData(llvm::raw_ostream& Out, PTHIdKey*, uint32_t pID,
                        unsigned) {
     ::Emit32(Out, pID);
   }        
@@ -654,10 +715,10 @@ std::pair<Offset,Offset> PTHWriter::EmitIdentifierTable() {
   //  (2) a map from (IdentifierInfo*, Offset)* -> persistent IDs
 
   // Note that we use 'calloc', so all the bytes are 0.
-  PCHIdKey* IIDMap = (PCHIdKey*) calloc(idcount, sizeof(PCHIdKey));
+  PTHIdKey* IIDMap = (PTHIdKey*) calloc(idcount, sizeof(PTHIdKey));
 
   // Create the hashtable.
-  OnDiskChainedHashTableGenerator<PCHIdentifierTableTrait> IIOffMap;
+  OnDiskChainedHashTableGenerator<PTHIdentifierTableTrait> IIOffMap;
   
   // Generate mapping from persistent IDs -> IdentifierInfo*.
   for (IDMap::iterator I=IM.begin(), E=IM.end(); I!=E; ++I) {

clang/include/clang/Lex/PTHManager.h

@@ -96,7 +96,7 @@ class PTHManager : public IdentifierInfoLookup {
   
 public:
   // The current PTH version.
-  enum { Version = 7 };
+  enum { Version = 8 };
 
   ~PTHManager();
   

clang/lib/Lex/PTHLexer.cpp

@@ -375,13 +375,15 @@ public:
   bool isEmpty() const { return NumEntries == 0; }
   
   class iterator {
+    internal_key_type key;
     const unsigned char* const data;
     const unsigned len;
   public:
     iterator() : data(0), len(0) {}
-    iterator(const unsigned char* d, unsigned l) : data(d), len(l) {}
+    iterator(const internal_key_type k, const unsigned char* d, unsigned l)
+      : key(k), data(d), len(l) {}
     
-    data_type operator*() const { return Info::ReadData(data, len); }    
+    data_type operator*() const { return Info::ReadData(key, data, len); }    
     bool operator==(const iterator& X) const { return X.data == data; }    
     bool operator!=(const iterator& X) const { return X.data != data; }
   };    
@@ -427,7 +429,7 @@ public:
       }
       
       // The key matches!
-      return iterator(Items + L.first, L.second);
+      return iterator(X, Items + L.first, L.second);
     }
     
     return iterator();
@@ -472,23 +474,22 @@ public:
   
 class VISIBILITY_HIDDEN PTHFileLookupCommonTrait {
 public:
-  typedef const char*      internal_key_type;
+  typedef std::pair<unsigned char, const char*> internal_key_type;
-  
+
-  static bool EqualKey(const char* a, const char* b) {
+  static unsigned ComputeHash(internal_key_type x) {
-    return strcmp(a, b) == 0;
+    return BernsteinHash(x.second);
-  }
-  
-  static unsigned ComputeHash(const char* x) {
-    return BernsteinHash(x);
   }
   
   static std::pair<unsigned, unsigned>
   ReadKeyDataLength(const unsigned char*& d) {
-    return std::make_pair((unsigned) ReadUnalignedLE16(d), 8U + (4+4+2+8+8));
+    unsigned keyLen = (unsigned) ReadUnalignedLE16(d);
+    unsigned dataLen = (unsigned) *(d++);
+    return std::make_pair(keyLen, dataLen);
   }
   
-  static const char* ReadKey(const unsigned char* d, unsigned) {
+  static internal_key_type ReadKey(const unsigned char* d, unsigned) {
-    return (const char*) d;
+    unsigned char k = *(d++); // Read the entry kind.
+    return std::make_pair(k, (const char*) d);
   }
 };
   
@@ -497,11 +498,17 @@ public:
   typedef const FileEntry* external_key_type;
   typedef PTHFileData      data_type;
   
-  static const char* GetInternalKey(const FileEntry* FE) {
+  static internal_key_type GetInternalKey(const FileEntry* FE) {
-    return FE->getName();
+    return std::make_pair((unsigned char) 0x1, FE->getName());
   }
+
+  static bool EqualKey(internal_key_type a, internal_key_type b) {
+    return a.first == b.first && strcmp(a.second, b.second) == 0;
+  }  
   
-  static PTHFileData ReadData(const unsigned char* d, unsigned) {
+  static PTHFileData ReadData(const internal_key_type& k, 
+                              const unsigned char* d, unsigned) {    
+    assert(k.first == 0x1 && "Only file lookups can match!");
     uint32_t x = ::ReadUnalignedLE32(d);
     uint32_t y = ::ReadUnalignedLE32(d);
     return PTHFileData(x, y); 
@@ -543,7 +550,8 @@ public:
       return std::make_pair((const char*) d, n-1);
     }
     
-  static uint32_t ReadData(const unsigned char* d, unsigned) {
+  static uint32_t ReadData(const internal_key_type& k, const unsigned char* d,
+                           unsigned) {
     return ::ReadUnalignedLE32(d);
   }
 };
@@ -768,6 +776,7 @@ PTHLexer *PTHManager::CreateLexer(FileID FID) {
 namespace {
 class VISIBILITY_HIDDEN PTHStatData {
 public:
+  const bool hasStat;
   const ino_t ino;
   const dev_t dev;
   const mode_t mode;
@@ -775,23 +784,40 @@ public:
   const off_t size;
   
   PTHStatData(ino_t i, dev_t d, mode_t mo, time_t m, off_t s)
-  : ino(i), dev(d), mode(mo), mtime(m), size(s) {}  
+  : hasStat(true), ino(i), dev(d), mode(mo), mtime(m), size(s) {}  
+  
+  PTHStatData()
+    : hasStat(false), ino(0), dev(0), mode(0), mtime(0), size(0) {}
 };
   
 class VISIBILITY_HIDDEN PTHStatLookupTrait : public PTHFileLookupCommonTrait {
 public:
-  typedef internal_key_type external_key_type;  // const char*
+  typedef const char* external_key_type;  // const char*
   typedef PTHStatData data_type;
     
-  static const char* GetInternalKey(external_key_type x) { return x; }
+  static internal_key_type GetInternalKey(const char *path) {
+    // The key 'kind' doesn't matter here because it is ignored in EqualKey.
+    return std::make_pair((unsigned char) 0x0, path);
+  }
+
+  static bool EqualKey(internal_key_type a, internal_key_type b) {
+    // When doing 'stat' lookups we don't care about the kind of 'a' and 'b',
+    // just the paths.
+    return strcmp(a.second, b.second) == 0;
+  }  
   
-  static data_type ReadData(const unsigned char* d, unsigned) {
+  static data_type ReadData(const internal_key_type& k, const unsigned char* d,
-    d += 4 * 2; // Skip the first 2 words.
+                            unsigned) {    
-    ino_t ino = (ino_t) ReadUnalignedLE32(d);
+    if (k.first == 0x1 /* File */) {
-    dev_t dev = (dev_t) ReadUnalignedLE32(d);
+      d += 4 * 2; // Skip the first 2 words.
-    mode_t mode = (mode_t) ReadUnalignedLE16(d);
+      ino_t ino = (ino_t) ReadUnalignedLE32(d);
-    time_t mtime = (time_t) ReadUnalignedLE64(d);    
+      dev_t dev = (dev_t) ReadUnalignedLE32(d);
-    return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
+      mode_t mode = (mode_t) ReadUnalignedLE16(d);
+      time_t mtime = (time_t) ReadUnalignedLE64(d);    
+      return data_type(ino, dev, mode, mtime, (off_t) ReadUnalignedLE64(d));
+    }
+    
+    return data_type();
   }
 };
 }