Skip to content

Remove realizationOutput type from backend #174

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
zombiezen merged 1 commit into from
Nov 17, 2025
Merged

Remove realizationOutput type from backend #174

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
162 changes: 59 additions & 103 deletions internal/backend/backend_store.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,158 +224,114 @@ func createTrustedPublicKeysTable(conn *sqlite.Conn, keys []*zbstore.Realization
}, nil
}

type realizationOutput struct {
path zbstore.Path
references map[zbstore.Path]sets.Set[equivalenceClass]
}

func recordRealizations(ctx context.Context, conn *sqlite.Conn, keyring *Keyring, drvHash nix.Hash, outputs map[string]realizationOutput) (err error) {
if log.IsEnabled(log.Debug) {
outputPaths := make(map[string]zbstore.Path)
for outputName, out := range outputs {
outputPaths[outputName] = out.path
}
log.Debugf(ctx, "Recording realizations for %v: %s", drvHash, formatOutputPaths(outputPaths))
}

func recordRealizations(conn *sqlite.Conn, realizations iter.Seq2[zbstore.RealizationOutputReference, *zbstore.Realization]) (err error) {
defer sqlitex.Save(conn)(&err)

if err := upsertDrvHash(conn, drvHash); err != nil {
return fmt.Errorf("record realizations for %v: %v", drvHash, err)
}
for _, output := range outputs {
if err := upsertPath(conn, output.path); err != nil {
return fmt.Errorf("record realizations for %v: %v", drvHash, err)
}
for path, eqClasses := range output.references {
if err := upsertPath(conn, path); err != nil {
return fmt.Errorf("record realizations for %v: %v", drvHash, err)
}
for eqClass := range eqClasses.All() {
h := eqClass.drvHashKey.toHash()
if err := upsertDrvHash(conn, h); err != nil {
return fmt.Errorf("record realizations for %v: %v", drvHash, err)
}
}
}
}

realizationStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert.sql")
if err != nil {
return fmt.Errorf("record realizations for %s: %v", drvHash, err)
return err
}
defer realizationStmt.Finalize()
refClassStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_reference_class.sql")
if err != nil {
return fmt.Errorf("record realizations for %s: %v", drvHash, err)
return err
}
defer refClassStmt.Finalize()
publicKeyStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_public_key.sql")
if err != nil {
return fmt.Errorf("record realizations for %s: %v", drvHash, err)
return err
}
defer publicKeyStmt.Finalize()
signatureStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_signature.sql")
if err != nil {
return fmt.Errorf("record realizations for %s: %v", drvHash, err)
return err
}
defer signatureStmt.Finalize()

realizationStmt.SetText(":drv_hash_algorithm", drvHash.Type().String())
realizationStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil))
refClassStmt.SetText(":referrer_drv_hash_algorithm", drvHash.Type().String())
refClassStmt.SetBytes(":referrer_drv_hash_bits", drvHash.Bytes(nil))
for outputName, output := range outputs {
realizationStmt.SetText(":output_name", outputName)
realizationStmt.SetText(":output_path", string(output.path))
for ref, realization := range realizations {
if err := upsertDrvHash(conn, ref.DerivationHash); err != nil {
return fmt.Errorf("record realization for %v: %v", ref, err)
}
if err := upsertPath(conn, realization.OutputPath); err != nil {
return fmt.Errorf("record realization for %v: %v", ref, err)
}

drvHashAlgorithm := ref.DerivationHash.Type().String()
drvHashBits := ref.DerivationHash.Bytes(nil)

realizationStmt.SetText(":drv_hash_algorithm", drvHashAlgorithm)
realizationStmt.SetBytes(":drv_hash_bits", drvHashBits)
realizationStmt.SetText(":output_name", ref.OutputName)
realizationStmt.SetText(":output_path", string(realization.OutputPath))
if _, err := realizationStmt.Step(); err != nil {
return fmt.Errorf("record realizations for %s: output %s: %v", drvHash, outputName, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}
if err := realizationStmt.Reset(); err != nil {
return fmt.Errorf("record realizations for %s: output %s: %v", drvHash, outputName, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}

refClassStmt.SetText(":referrer_path", string(output.path))
refClassStmt.SetText(":referrer_output_name", outputName)
for path, eqClasses := range output.references {
refClassStmt.SetText(":reference_path", string(path))
for eqClass := range eqClasses.All() {
if eqClass.isZero() {
refClassStmt.SetNull(":reference_drv_hash_algorithm")
refClassStmt.SetNull(":reference_drv_hash_bits")
refClassStmt.SetNull(":reference_output_name")
} else {
h := eqClass.drvHashKey.toHash()
refClassStmt.SetText(":reference_drv_hash_algorithm", h.Type().String())
refClassStmt.SetBytes(":reference_drv_hash_bits", h.Bytes(nil))
refClassStmt.SetText(":reference_output_name", eqClass.outputName.Value())
}
refClassStmt.SetText(":referrer_drv_hash_algorithm", drvHashAlgorithm)
refClassStmt.SetBytes(":referrer_drv_hash_bits", drvHashBits)
refClassStmt.SetText(":referrer_path", string(realization.OutputPath))
refClassStmt.SetText(":referrer_output_name", ref.OutputName)
for _, rc := range realization.ReferenceClasses {
if err := upsertPath(conn, rc.Path); err != nil {
return fmt.Errorf("record realizations for %v: reference %s: %v", ref, rc.Path, err)
}

if _, err := refClassStmt.Step(); err != nil {
return fmt.Errorf("record realizations for %s: output %s: reference %s: %v", drvHash, outputName, path, err)
}
if err := refClassStmt.Reset(); err != nil {
return fmt.Errorf("record realizations for %s: output %s: reference %s: %v", drvHash, outputName, path, err)
refClassStmt.SetText(":reference_path", string(rc.Path))
if !rc.Realization.Valid {
refClassStmt.SetNull(":reference_drv_hash_algorithm")
refClassStmt.SetNull(":reference_drv_hash_bits")
refClassStmt.SetNull(":reference_output_name")
} else {
h := rc.Realization.X.DerivationHash
if err := upsertDrvHash(conn, h); err != nil {
return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err)
}

refClassStmt.SetText(":reference_drv_hash_algorithm", h.Type().String())
refClassStmt.SetBytes(":reference_drv_hash_bits", h.Bytes(nil))
refClassStmt.SetText(":reference_output_name", rc.Realization.X.OutputName)
}
}

ref := zbstore.RealizationOutputReference{
DerivationHash: drvHash,
OutputName: outputName,
}
signatures, err := keyring.Sign(ref, makeRealization(output))
if err != nil {
log.Warnf(ctx, "%v", err)
if _, err := refClassStmt.Step(); err != nil {
return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err)
}
if err := refClassStmt.Reset(); err != nil {
return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err)
}
}
for _, sig := range signatures {

for _, sig := range realization.Signatures {
publicKeyStmt.SetText(":format", string(sig.PublicKey.Format))
publicKeyStmt.SetBytes(":public_key", sig.PublicKey.Data)
if _, err := publicKeyStmt.Step(); err != nil {
return fmt.Errorf("record realizations for %v: %v", ref, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}
if err := publicKeyStmt.Reset(); err != nil {
return fmt.Errorf("record realizations for %v: %v", ref, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}

signatureStmt.SetText(":drv_hash_algorithm", drvHash.Type().String())
signatureStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil))
signatureStmt.SetText(":output_name", outputName)
signatureStmt.SetText(":output_path", string(output.path))
signatureStmt.SetText(":drv_hash_algorithm", drvHashAlgorithm)
signatureStmt.SetBytes(":drv_hash_bits", drvHashBits)
signatureStmt.SetText(":output_name", ref.OutputName)
signatureStmt.SetText(":output_path", string(realization.OutputPath))
signatureStmt.SetText(":format", string(sig.PublicKey.Format))
signatureStmt.SetBytes(":public_key", sig.PublicKey.Data)
signatureStmt.SetBytes(":signature", sig.Signature)
if _, err := signatureStmt.Step(); err != nil {
return fmt.Errorf("record realizations for %v: %v", ref, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}
if err := signatureStmt.Reset(); err != nil {
return fmt.Errorf("record realizations for %v: %v", ref, err)
return fmt.Errorf("record realization for %v: %v", ref, err)
}
}
}

return nil
}

func makeRealization(output realizationOutput) *zbstore.Realization {
r := &zbstore.Realization{
OutputPath: output.path,
}
for path, eqClasses := range output.references {
for eqClass := range eqClasses.All() {
rc := &zbstore.ReferenceClass{Path: path}
if !eqClass.isZero() {
rc.Realization = zbstore.NonNull(zbstore.RealizationOutputReference{
DerivationHash: eqClass.drvHashKey.toHash(),
OutputName: eqClass.outputName.Value(),
})
}
r.ReferenceClasses = append(r.ReferenceClasses, rc)
}
}
return r
}

// pathInfo returns basic information about an object in the store.
func pathInfo(conn *sqlite.Conn, path zbstore.Path) (_ *ObjectInfo, err error) {
defer sqlitex.Save(conn)(&err)
Expand Down
11 changes: 11 additions & 0 deletions internal/backend/equivalence_class.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,21 @@ func newEquivalenceClass(drvHash nix.Hash, outputName string) equivalenceClass {
}
}

func realizationOutputReferenceKey(ref zbstore.RealizationOutputReference) equivalenceClass {
return newEquivalenceClass(ref.DerivationHash, ref.OutputName)
}

func (eqClass equivalenceClass) isZero() bool {
return eqClass == equivalenceClass{}
}

func (eqClass equivalenceClass) toRealizationOutputReference() zbstore.RealizationOutputReference {
return zbstore.RealizationOutputReference{
DerivationHash: eqClass.drvHashKey.toHash(),
OutputName: eqClass.outputName.Value(),
}
}

func (eqClass equivalenceClass) String() string {
if eqClass.isZero() {
return "ε"
Expand Down
80 changes: 51 additions & 29 deletions internal/backend/realize.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -819,13 +819,18 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets
_, err = os.Lstat(b.server.realPath(outputPath))
log.Debugf(ctx, "%s exists=%t (output of %s)", outputPath, err == nil, drvPath)
if err == nil {
outputs := map[string]realizationOutput{
zbstore.DefaultDerivationOutputName: {
path: outputPath,
// Fixed outputs don't have references.
outputs := zbstore.RealizationMap{
DerivationHash: drvHash,
Realizations: map[string][]*zbstore.Realization{
zbstore.DefaultDerivationOutputName: {
{
OutputPath: outputPath,
// Fixed outputs don't have references.
},
},
},
}
if err := b.recordRealizations(ctx, conn, drvHash, buildResultID, outputs); err != nil {
if err := b.recordRealizations(ctx, conn, buildResultID, outputs); err != nil {
return fmt.Errorf("build %s: %v", drvPath, err)
}
return nil
Expand Down Expand Up @@ -906,7 +911,10 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets
return err
}
inputPaths := sets.CollectSorted(maps.Keys(inputs))
outputs := make(map[string]realizationOutput)
outputs := zbstore.RealizationMap{
DerivationHash: drvHash,
Realizations: make(map[string][]*zbstore.Realization),
}
for outputName, tempOutputPath := range tempOutPaths {
ref := zbstore.OutputReference{
DrvPath: drvPath,
Expand All @@ -926,28 +934,35 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets
drvPath, outputName, info.StorePath, prev.path)
}

outputs[outputName] = realizationOutput{
path: info.StorePath,
references: maps.Collect(func(yield func(zbstore.Path, sets.Set[equivalenceClass]) bool) {
for ref, eqClasses := range inputs {
if info.References.Has(ref) {
if !yield(ref, eqClasses) {
return
}
}
r := &zbstore.Realization{OutputPath: info.StorePath}
for ref, eqClasses := range inputs {
if info.References.Has(ref) {
for eqClass := range eqClasses.All() {
r.ReferenceClasses = append(r.ReferenceClasses, &zbstore.ReferenceClass{
Path: ref,
Realization: zbstore.NonNull(eqClass.toRealizationOutputReference()),
})
}
}),
}
}
r.Signatures, err = b.server.keyring.Sign(zbstore.RealizationOutputReference{
DerivationHash: drvHash,
OutputName: outputName,
}, r)
if err != nil {
log.Warnf(ctx, "Signing built realization: %v", err)
}
outputs.Realizations[outputName] = []*zbstore.Realization{r}
}

// Record realizations.
if err := b.recordRealizations(ctx, conn, drvHash, buildResultID, outputs); err != nil {
if err := b.recordRealizations(ctx, conn, buildResultID, outputs); err != nil {
return fmt.Errorf("build %s: %v", drvPath, err)
}

log.Infof(ctx, "Built %s: %s", drvPath, formatOutputPaths(maps.Collect(func(yield func(string, zbstore.Path) bool) {
for outputName, r := range outputs {
if !yield(outputName, r.path) {
for ref, r := range outputs.All() {
if !yield(ref.OutputName, r.OutputPath) {
return
}
}
Expand Down Expand Up @@ -1623,19 +1638,26 @@ func rewriteAtPath(path string, baseOffset int64, newDigest string, rewriters []
// recordRealizations calls [recordRealizations] and [recordBuildOutputs] in a transaction
// and on success, saves the realizations into b.realizations.
// The outputs must exist in the store.
func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drvHash nix.Hash, buildResultID int64, outputs map[string]realizationOutput) (err error) {
func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, buildResultID int64, outputs zbstore.RealizationMap) (err error) {
if log.IsEnabled(log.Debug) {
outputPaths := make(map[string]zbstore.Path)
for ref, r := range outputs.All() {
outputPaths[ref.OutputName] = r.OutputPath
}
log.Debugf(ctx, "Recording realizations for %v: %s", outputs.DerivationHash, formatOutputPaths(outputPaths))
}

endFn, err := sqlitex.ImmediateTransaction(conn)
if err != nil {
return fmt.Errorf("record realizations for %v: %v", drvHash, err)
return fmt.Errorf("record realizations for %v: %v", outputs.DerivationHash, err)
}
defer endFn(&err)

if err := recordRealizations(ctx, conn, b.server.keyring, drvHash, outputs); err != nil {
if err := recordRealizations(conn, outputs.All()); err != nil {
return err
}
buildOutputs := func(yield func(string, zbstore.Path) bool) {
for outputName, output := range outputs {
if !yield(outputName, output.path) {
for ref, r := range outputs.All() {
if !yield(ref.OutputName, r.OutputPath) {
return
}
}
Expand All @@ -1644,10 +1666,10 @@ func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drv
return err
}

for outputName, output := range outputs {
eqClass := newEquivalenceClass(drvHash, outputName)
for ref, r := range outputs.All() {
closure := make(map[zbstore.Path]sets.Set[equivalenceClass])
pe := pathAndEquivalenceClass{path: output.path, equivalenceClass: eqClass}
eqClass := realizationOutputReferenceKey(ref)
pe := pathAndEquivalenceClass{path: r.OutputPath, equivalenceClass: eqClass}
err := closurePaths(conn, pe, func(pe pathAndEquivalenceClass) bool {
addToMultiMap(closure, pe.path, pe.equivalenceClass)
return true
Expand All @@ -1656,7 +1678,7 @@ func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drv
return err
}
b.realizations[eqClass] = cachedRealization{
path: output.path,
path: r.OutputPath,
closure: closure,
}
}
Expand Down