shithub: furgit

ref: a902f6849d938a51fa34dc292f5c81d26c4b6c36
dir: /format/packfile/ingest/idx_write.go/

package ingest

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"hash"
	"io"
	"slices"

	"codeberg.org/lindenii/furgit/internal/intconv"
	"codeberg.org/lindenii/furgit/internal/progress"
)

const (
	idxMagicV2   = 0xff744f63
	idxVersionV2 = 2
)

// writeIdx writes idx v2 for resolved records.
func writeIdx(state *ingestState) error {
	order := buildIdxOrder(state)

	hashImpl, err := state.algo.New()
	if err != nil {
		return err
	}

	write := func(src []byte) error {
		_, writeErr := state.idxFile.Write(src)
		if writeErr != nil {
			return writeErr
		}

		_, writeErr = hashImpl.Write(src)
		if writeErr != nil {
			return writeErr
		}

		return nil
	}

	var (
		scratch [8]byte
		fanout  [256]uint32
	)

	writeProgressf(state, "writing index fanout...\r")

	for _, recordIdx := range order {
		idRaw := state.records[recordIdx].objectID.Bytes()
		fanout[idRaw[0]]++
	}

	binary.BigEndian.PutUint32(scratch[:4], idxMagicV2)
	binary.BigEndian.PutUint32(scratch[4:8], idxVersionV2)

	err = write(scratch[:8])
	if err != nil {
		return err
	}

	var cumulative uint32
	for i := range fanout {
		cumulative += fanout[i]
		binary.BigEndian.PutUint32(scratch[:4], cumulative)

		err := write(scratch[:4])
		if err != nil {
			return err
		}
	}

	writeProgressf(state, "writing index fanout: done.\n")

	largeOffsetCount := 0

	for idx := range state.records {
		if state.records[idx].offset >= 0x80000000 {
			largeOffsetCount++
		}
	}

	oidMeter := progress.New(progress.Options{
		Writer: state.opts.Progress,
		Flush:  state.opts.ProgressFlush,
		Title:  "writing index object ids",
		Total:  uint64(len(order)),
	})

	var oidDone uint64

	for _, recordIdx := range order {
		idRaw := state.records[recordIdx].objectID.Bytes()

		err := write(idRaw)
		if err != nil {
			return err
		}

		oidDone++
		oidMeter.Set(oidDone, 0)
	}

	if oidDone > 0 {
		oidMeter.Stop("done")
	}

	crcMeter := progress.New(progress.Options{
		Writer: state.opts.Progress,
		Flush:  state.opts.ProgressFlush,
		Title:  "writing index crc32",
		Total:  uint64(len(order)),
	})

	var crcDone uint64

	for _, recordIdx := range order {
		binary.BigEndian.PutUint32(scratch[:4], state.records[recordIdx].crc32)

		err := write(scratch[:4])
		if err != nil {
			return err
		}

		crcDone++
		crcMeter.Set(crcDone, 0)
	}

	if crcDone > 0 {
		crcMeter.Stop("done")
	}

	largeOffsets := make([]uint64, 0)
	offsetMeter := progress.New(progress.Options{
		Writer: state.opts.Progress,
		Flush:  state.opts.ProgressFlush,
		Title:  "writing index offsets",
		Total:  uint64(len(order)),
	})

	var offsetDone uint64

	for _, recordIdx := range order {
		offset := state.records[recordIdx].offset
		if offset >= 0x80000000 {
			largeOffsetIdx, err := intconv.IntToUint32(len(largeOffsets))
			if err != nil {
				return err
			}

			word := 0x80000000 | largeOffsetIdx

			largeOffsets = append(largeOffsets, offset)

			binary.BigEndian.PutUint32(scratch[:4], word)
		} else {
			binary.BigEndian.PutUint32(scratch[:4], uint32(offset))
		}

		err := write(scratch[:4])
		if err != nil {
			return err
		}

		offsetDone++
		offsetMeter.Set(offsetDone, 0)
	}

	if offsetDone > 0 {
		offsetMeter.Stop("done")
	}

	total, err := intconv.IntToUint64(largeOffsetCount)
	if err != nil {
		return err
	}

	largeOffsetMeter := progress.New(progress.Options{
		Writer: state.opts.Progress,
		Flush:  state.opts.ProgressFlush,
		Title:  "writing index large offsets",
		Total:  total,
	})

	var largeOffsetDone uint64

	for _, off := range largeOffsets {
		binary.BigEndian.PutUint64(scratch[:8], off)

		err := write(scratch[:8])
		if err != nil {
			return err
		}

		largeOffsetDone++
		largeOffsetMeter.Set(largeOffsetDone, 0)
	}

	if largeOffsetDone > 0 {
		largeOffsetMeter.Stop("done")
	}

	writeProgressf(state, "writing index trailer...\r")

	err = write(state.packHash.Bytes())
	if err != nil {
		return err
	}

	idxHash := hashImpl.Sum(nil)

	_, err = state.idxFile.Write(idxHash)
	if err != nil {
		return err
	}

	err = state.idxFile.Sync()
	if err != nil {
		return err
	}

	writeProgressf(state, "writing index trailer: done.\n")

	return nil
}

// buildIdxOrder returns record indexes sorted by ObjectID.
func buildIdxOrder(state *ingestState) []int {
	out := make([]int, 0, len(state.records))
	for idx := range state.records {
		out = append(out, idx)
	}

	slices.SortFunc(out, func(a, b int) int {
		return bytes.Compare(state.records[a].objectID.Bytes(), state.records[b].objectID.Bytes())
	})

	return out
}

// verifyResolvedRecords checks that all records are fully resolved before index writing.
func verifyResolvedRecords(state *ingestState) error {
	for idx, record := range state.records {
		if !record.resolved {
			return fmt.Errorf("packfile/ingest: unresolved record %d at offset %d", idx, record.offset)
		}
	}

	return nil
}

// writeAndHash writes src to dst and updates hash.
func writeAndHash(dst io.Writer, hashImpl hash.Hash, src []byte) error {
	_, err := dst.Write(src)
	if err != nil {
		return err
	}

	_, err = hashImpl.Write(src)
	if err != nil {
		return err
	}

	return nil
}