ref: 5eda091d68a427e9f23e120bad1767f796ae58b6
parent: cc1af64ae2908cdbc4c65a54950ec6dfc6293455
author: Runxi Yu <me@runxiyu.org>
date: Sat Feb 21 14:42:08 EST 2026
objectstore/packed: Lazily parse idx metadata
--- a/objectstore/packed/idx_load.go
+++ b/objectstore/packed/idx_load.go
@@ -17,6 +17,16 @@
offset uint64
}
+// packCandidate describes one discovered pack/index pair.
+type packCandidate struct {+ // packName is the .pack basename.
+ packName string
+ // idxName is the .idx basename.
+ idxName string
+ // mtime is the pack file modification time for initial ordering.
+ mtime int64
+}
+
// idxFile stores one mapped and validated idx v2 file.
type idxFile struct {// idxName is the basename of this .idx file.
@@ -46,8 +56,30 @@
offset64Count int
}
-// loadIndexes loads and validates all .idx files under objects/pack.
-func (store *Store) loadIndexes() ([]*idxFile, error) {+// ensureCandidates discovers pack/index pairs once.
+func (store *Store) ensureCandidates() error {+ store.discoverOnce.Do(func() {+ candidates, err := store.discoverCandidates()
+ candidateByPack := make(map[string]packCandidate, len(candidates))
+ for _, candidate := range candidates {+ candidateByPack[candidate.packName] = candidate
+ }
+ store.stateMu.Lock()
+ store.candidates = candidates
+ store.candidateByPack = candidateByPack
+ store.discoverErr = err
+ store.stateMu.Unlock()
+ })
+
+ store.stateMu.RLock()
+ err := store.discoverErr
+ store.stateMu.RUnlock()
+ return err
+}
+
+// discoverCandidates scans the objects/pack root and returns sorted pack/index
+// pairs.
+func (store *Store) discoverCandidates() ([]packCandidate, error) { dir, err := store.root.Open(".") if err != nil { if os.IsNotExist(err) {@@ -56,39 +88,97 @@
return nil, err
}
defer func() { _ = dir.Close() }()+
entries, err := dir.ReadDir(-1)
if err != nil {return nil, err
}
- idxNames := make([]string, 0, len(entries))
+ candidates := make([]packCandidate, 0, len(entries))
for _, entry := range entries { if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".idx") {continue
}
- idxNames = append(idxNames, entry.Name())
- }
- slices.Sort(idxNames)
- out := make([]*idxFile, 0, len(idxNames))
- for _, idxName := range idxNames {+ idxName := entry.Name()
packName := strings.TrimSuffix(idxName, ".idx") + ".pack"
- if _, err := store.root.Stat(packName); err != nil {+ packInfo, err := store.root.Stat(packName)
+ if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("objectstore/packed: missing pack file for index %q", idxName)}
return nil, err
}
- index, err := openIdxFile(store.root, idxName, packName, store.algo)
- if err != nil {- for _, loaded := range out {- _ = loaded.close()
+
+ candidates = append(candidates, packCandidate{+ packName: packName,
+ idxName: idxName,
+ mtime: packInfo.ModTime().UnixNano(),
+ })
+ }
+
+ slices.SortFunc(candidates, func(a, b packCandidate) int {+ if a.mtime != b.mtime {+ if a.mtime > b.mtime {+ return -1
}
- return nil, err
+ return 1
}
- out = append(out, index)
+ return strings.Compare(a.packName, b.packName)
+ })
+
+ return candidates, nil
+}
+
+// candidateForPack returns one discovered candidate for a pack basename.
+func (store *Store) candidateForPack(packName string) (packCandidate, bool) {+ store.stateMu.RLock()
+ candidate, ok := store.candidateByPack[packName]
+ store.stateMu.RUnlock()
+ return candidate, ok
+}
+
+// openIndex returns one opened and parsed index, caching it by pack basename.
+func (store *Store) openIndex(candidate packCandidate) (*idxFile, error) {+ store.stateMu.RLock()
+ if index, ok := store.idxByPack[candidate.packName]; ok {+ store.stateMu.RUnlock()
+ return index, nil
}
- return out, nil
+ store.stateMu.RUnlock()
+
+ index, err := openIdxFile(store.root, candidate.idxName, candidate.packName, store.algo)
+ if err != nil {+ return nil, err
+ }
+
+ store.stateMu.Lock()
+ if existing, ok := store.idxByPack[candidate.packName]; ok {+ store.stateMu.Unlock()
+ _ = index.close()
+ return existing, nil
+ }
+ store.idxByPack[candidate.packName] = index
+ store.stateMu.Unlock()
+ return index, nil
+}
+
+// touchCandidate moves one candidate to the front of the lookup order.
+func (store *Store) touchCandidate(packName string) {+ store.stateMu.Lock()
+ defer store.stateMu.Unlock()
+ for i := range store.candidates {+ if store.candidates[i].packName != packName {+ continue
+ }
+ if i == 0 {+ return
+ }
+ candidate := store.candidates[i]
+ copy(store.candidates[1:i+1], store.candidates[0:i])
+ store.candidates[0] = candidate
+ return
+ }
}
// openIdxFile maps and validates one idx v2 file.
--- a/objectstore/packed/store.go
+++ b/objectstore/packed/store.go
@@ -21,16 +21,16 @@
// algo is the expected object ID algorithm for lookups.
algo objectid.Algorithm
- // loadOnce guards one-time index loading.
- loadOnce sync.Once
- // loadErr stores index loading failures.
- loadErr error
- // indexesLoaded reports whether indexes/loadErr have been initialized.
- indexesLoaded bool
- // indexes stores parsed .idx handles.
- indexes []*idxFile
- // indexByPack maps one pack basename to its parsed index.
- indexByPack map[string]*idxFile
+ // discoverOnce guards one-time pack candidate discovery.
+ discoverOnce sync.Once
+ // discoverErr stores candidate discovery failures.
+ discoverErr error
+ // candidates stores known pack/index pairs in lookup priority order.
+ candidates []packCandidate
+ // candidateByPack maps pack basename to discovered candidate.
+ candidateByPack map[string]packCandidate
+ // idxByPack caches opened and parsed indexes by pack basename.
+ idxByPack map[string]*idxFile
// stateMu guards index publication, pack cache, and close state.
stateMu sync.RWMutex
@@ -54,10 +54,12 @@
return nil, objectid.ErrInvalidAlgorithm
}
return &Store{- root: root,
- algo: algo,
- packs: make(map[string]*packFile),
- deltaCache: newDeltaCache(defaultDeltaCacheMaxBytes),
+ root: root,
+ algo: algo,
+ candidateByPack: make(map[string]packCandidate),
+ idxByPack: make(map[string]*idxFile),
+ packs: make(map[string]*packFile),
+ deltaCache: newDeltaCache(defaultDeltaCacheMaxBytes),
}, nil
}
@@ -71,7 +73,7 @@
store.closed = true
root := store.root
packs := store.packs
- indexes := store.indexes
+ indexes := store.idxByPack
store.stateMu.Unlock()
var closeErr error
@@ -81,9 +83,6 @@
}
}
for _, index := range indexes {- if index == nil {- continue
- }
if err := index.close(); err != nil && closeErr == nil {closeErr = err
}
@@ -98,30 +97,6 @@
return closeErr
}
-// ensureIndexes loads and validates all pack indexes once.
-func (store *Store) ensureIndexes() error {- store.loadOnce.Do(func() {- indexes, err := store.loadIndexes()
- indexByPack := make(map[string]*idxFile, len(indexes))
- for _, index := range indexes {- indexByPack[index.packName] = index
- }
- store.stateMu.Lock()
- store.indexes = indexes
- store.indexByPack = indexByPack
- store.loadErr = err
- store.indexesLoaded = true
- store.stateMu.Unlock()
- })
-
- store.stateMu.RLock()
- defer store.stateMu.RUnlock()
- if store.indexesLoaded {- return store.loadErr
- }
- return errors.New("objectstore/packed: indexes were not initialized")-}
-
// lookup resolves one object ID to its pack location.
func (store *Store) lookup(id objectid.ObjectID) (location, error) {var zero location
@@ -128,15 +103,25 @@
if id.Algorithm() != store.algo { return zero, errors.New("objectstore/packed: object id algorithm mismatch")}
- if err := store.ensureIndexes(); err != nil {+ if err := store.ensureCandidates(); err != nil {return zero, err
}
- for _, index := range store.indexes {+
+ store.stateMu.RLock()
+ candidates := append([]packCandidate(nil), store.candidates...)
+ store.stateMu.RUnlock()
+
+ for _, candidate := range candidates {+ index, err := store.openIndex(candidate)
+ if err != nil {+ return zero, err
+ }
offset, ok, err := index.lookup(id)
if err != nil {return zero, err
}
if ok {+ store.touchCandidate(candidate.packName)
return location{packName: index.packName, offset: offset}, nil}
}
@@ -185,16 +170,16 @@
// verifyPackMatchesIndexes checks that one opened pack's trailer hash matches
// every loaded index that references the same pack name.
func (store *Store) verifyPackMatchesIndexes(pack *packFile) error {- store.stateMu.RLock()
- index := store.indexByPack[pack.name]
- indexesLoaded := store.indexesLoaded
- store.stateMu.RUnlock()
-
- if !indexesLoaded {- return nil
+ if err := store.ensureCandidates(); err != nil {+ return err
}
- if index == nil {- return nil
+ candidate, ok := store.candidateForPack(pack.name)
+ if !ok {+ return fmt.Errorf("objectstore/packed: missing index for pack %q", pack.name)+ }
+ index, err := store.openIndex(candidate)
+ if err != nil {+ return err
}
if err := packchecksum.VerifyPackMatchesIdx(pack.data, index.data, store.algo); err != nil { return fmt.Errorf("objectstore/packed: pack %q does not match idx %q: %w", pack.name, index.idxName, err)--
⑨