diff --git a/internal/backend/backend_store.go b/internal/backend/backend_store.go index b1858cc..f720be4 100644 --- a/internal/backend/backend_store.go +++ b/internal/backend/backend_store.go @@ -224,132 +224,107 @@ func createTrustedPublicKeysTable(conn *sqlite.Conn, keys []*zbstore.Realization }, nil } -type realizationOutput struct { - path zbstore.Path - references map[zbstore.Path]sets.Set[equivalenceClass] -} - -func recordRealizations(ctx context.Context, conn *sqlite.Conn, keyring *Keyring, drvHash nix.Hash, outputs map[string]realizationOutput) (err error) { - if log.IsEnabled(log.Debug) { - outputPaths := make(map[string]zbstore.Path) - for outputName, out := range outputs { - outputPaths[outputName] = out.path - } - log.Debugf(ctx, "Recording realizations for %v: %s", drvHash, formatOutputPaths(outputPaths)) - } - +func recordRealizations(conn *sqlite.Conn, realizations iter.Seq2[zbstore.RealizationOutputReference, *zbstore.Realization]) (err error) { defer sqlitex.Save(conn)(&err) - if err := upsertDrvHash(conn, drvHash); err != nil { - return fmt.Errorf("record realizations for %v: %v", drvHash, err) - } - for _, output := range outputs { - if err := upsertPath(conn, output.path); err != nil { - return fmt.Errorf("record realizations for %v: %v", drvHash, err) - } - for path, eqClasses := range output.references { - if err := upsertPath(conn, path); err != nil { - return fmt.Errorf("record realizations for %v: %v", drvHash, err) - } - for eqClass := range eqClasses.All() { - h := eqClass.drvHashKey.toHash() - if err := upsertDrvHash(conn, h); err != nil { - return fmt.Errorf("record realizations for %v: %v", drvHash, err) - } - } - } - } - realizationStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert.sql") if err != nil { - return fmt.Errorf("record realizations for %s: %v", drvHash, err) + return err } defer realizationStmt.Finalize() refClassStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_reference_class.sql") if err != nil { - return fmt.Errorf("record realizations for %s: %v", drvHash, err) + return err } defer refClassStmt.Finalize() publicKeyStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_public_key.sql") if err != nil { - return fmt.Errorf("record realizations for %s: %v", drvHash, err) + return err } defer publicKeyStmt.Finalize() signatureStmt, err := sqlitex.PrepareTransientFS(conn, sqlFiles(), "realizations/insert_signature.sql") if err != nil { - return fmt.Errorf("record realizations for %s: %v", drvHash, err) + return err } defer signatureStmt.Finalize() - realizationStmt.SetText(":drv_hash_algorithm", drvHash.Type().String()) - realizationStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil)) - refClassStmt.SetText(":referrer_drv_hash_algorithm", drvHash.Type().String()) - refClassStmt.SetBytes(":referrer_drv_hash_bits", drvHash.Bytes(nil)) - for outputName, output := range outputs { - realizationStmt.SetText(":output_name", outputName) - realizationStmt.SetText(":output_path", string(output.path)) + for ref, realization := range realizations { + if err := upsertDrvHash(conn, ref.DerivationHash); err != nil { + return fmt.Errorf("record realization for %v: %v", ref, err) + } + if err := upsertPath(conn, realization.OutputPath); err != nil { + return fmt.Errorf("record realization for %v: %v", ref, err) + } + + drvHashAlgorithm := ref.DerivationHash.Type().String() + drvHashBits := ref.DerivationHash.Bytes(nil) + + realizationStmt.SetText(":drv_hash_algorithm", drvHashAlgorithm) + realizationStmt.SetBytes(":drv_hash_bits", drvHashBits) + realizationStmt.SetText(":output_name", ref.OutputName) + realizationStmt.SetText(":output_path", string(realization.OutputPath)) if _, err := realizationStmt.Step(); err != nil { - return fmt.Errorf("record realizations for %s: output %s: %v", drvHash, outputName, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } if err := realizationStmt.Reset(); err != nil { - return fmt.Errorf("record realizations for %s: output %s: %v", drvHash, outputName, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } - refClassStmt.SetText(":referrer_path", string(output.path)) - refClassStmt.SetText(":referrer_output_name", outputName) - for path, eqClasses := range output.references { - refClassStmt.SetText(":reference_path", string(path)) - for eqClass := range eqClasses.All() { - if eqClass.isZero() { - refClassStmt.SetNull(":reference_drv_hash_algorithm") - refClassStmt.SetNull(":reference_drv_hash_bits") - refClassStmt.SetNull(":reference_output_name") - } else { - h := eqClass.drvHashKey.toHash() - refClassStmt.SetText(":reference_drv_hash_algorithm", h.Type().String()) - refClassStmt.SetBytes(":reference_drv_hash_bits", h.Bytes(nil)) - refClassStmt.SetText(":reference_output_name", eqClass.outputName.Value()) - } + refClassStmt.SetText(":referrer_drv_hash_algorithm", drvHashAlgorithm) + refClassStmt.SetBytes(":referrer_drv_hash_bits", drvHashBits) + refClassStmt.SetText(":referrer_path", string(realization.OutputPath)) + refClassStmt.SetText(":referrer_output_name", ref.OutputName) + for _, rc := range realization.ReferenceClasses { + if err := upsertPath(conn, rc.Path); err != nil { + return fmt.Errorf("record realizations for %v: reference %s: %v", ref, rc.Path, err) + } - if _, err := refClassStmt.Step(); err != nil { - return fmt.Errorf("record realizations for %s: output %s: reference %s: %v", drvHash, outputName, path, err) - } - if err := refClassStmt.Reset(); err != nil { - return fmt.Errorf("record realizations for %s: output %s: reference %s: %v", drvHash, outputName, path, err) + refClassStmt.SetText(":reference_path", string(rc.Path)) + if !rc.Realization.Valid { + refClassStmt.SetNull(":reference_drv_hash_algorithm") + refClassStmt.SetNull(":reference_drv_hash_bits") + refClassStmt.SetNull(":reference_output_name") + } else { + h := rc.Realization.X.DerivationHash + if err := upsertDrvHash(conn, h); err != nil { + return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err) } + + refClassStmt.SetText(":reference_drv_hash_algorithm", h.Type().String()) + refClassStmt.SetBytes(":reference_drv_hash_bits", h.Bytes(nil)) + refClassStmt.SetText(":reference_output_name", rc.Realization.X.OutputName) } - } - ref := zbstore.RealizationOutputReference{ - DerivationHash: drvHash, - OutputName: outputName, - } - signatures, err := keyring.Sign(ref, makeRealization(output)) - if err != nil { - log.Warnf(ctx, "%v", err) + if _, err := refClassStmt.Step(); err != nil { + return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err) + } + if err := refClassStmt.Reset(); err != nil { + return fmt.Errorf("record realization for %v: reference %s: %v", ref, rc.Path, err) + } } - for _, sig := range signatures { + + for _, sig := range realization.Signatures { publicKeyStmt.SetText(":format", string(sig.PublicKey.Format)) publicKeyStmt.SetBytes(":public_key", sig.PublicKey.Data) if _, err := publicKeyStmt.Step(); err != nil { - return fmt.Errorf("record realizations for %v: %v", ref, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } if err := publicKeyStmt.Reset(); err != nil { - return fmt.Errorf("record realizations for %v: %v", ref, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } - signatureStmt.SetText(":drv_hash_algorithm", drvHash.Type().String()) - signatureStmt.SetBytes(":drv_hash_bits", drvHash.Bytes(nil)) - signatureStmt.SetText(":output_name", outputName) - signatureStmt.SetText(":output_path", string(output.path)) + signatureStmt.SetText(":drv_hash_algorithm", drvHashAlgorithm) + signatureStmt.SetBytes(":drv_hash_bits", drvHashBits) + signatureStmt.SetText(":output_name", ref.OutputName) + signatureStmt.SetText(":output_path", string(realization.OutputPath)) signatureStmt.SetText(":format", string(sig.PublicKey.Format)) signatureStmt.SetBytes(":public_key", sig.PublicKey.Data) signatureStmt.SetBytes(":signature", sig.Signature) if _, err := signatureStmt.Step(); err != nil { - return fmt.Errorf("record realizations for %v: %v", ref, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } if err := signatureStmt.Reset(); err != nil { - return fmt.Errorf("record realizations for %v: %v", ref, err) + return fmt.Errorf("record realization for %v: %v", ref, err) } } } @@ -357,25 +332,6 @@ func recordRealizations(ctx context.Context, conn *sqlite.Conn, keyring *Keyring return nil } -func makeRealization(output realizationOutput) *zbstore.Realization { - r := &zbstore.Realization{ - OutputPath: output.path, - } - for path, eqClasses := range output.references { - for eqClass := range eqClasses.All() { - rc := &zbstore.ReferenceClass{Path: path} - if !eqClass.isZero() { - rc.Realization = zbstore.NonNull(zbstore.RealizationOutputReference{ - DerivationHash: eqClass.drvHashKey.toHash(), - OutputName: eqClass.outputName.Value(), - }) - } - r.ReferenceClasses = append(r.ReferenceClasses, rc) - } - } - return r -} - // pathInfo returns basic information about an object in the store. func pathInfo(conn *sqlite.Conn, path zbstore.Path) (_ *ObjectInfo, err error) { defer sqlitex.Save(conn)(&err) diff --git a/internal/backend/equivalence_class.go b/internal/backend/equivalence_class.go index 67fc0e3..56b91c0 100644 --- a/internal/backend/equivalence_class.go +++ b/internal/backend/equivalence_class.go @@ -30,10 +30,21 @@ func newEquivalenceClass(drvHash nix.Hash, outputName string) equivalenceClass { } } +func realizationOutputReferenceKey(ref zbstore.RealizationOutputReference) equivalenceClass { + return newEquivalenceClass(ref.DerivationHash, ref.OutputName) +} + func (eqClass equivalenceClass) isZero() bool { return eqClass == equivalenceClass{} } +func (eqClass equivalenceClass) toRealizationOutputReference() zbstore.RealizationOutputReference { + return zbstore.RealizationOutputReference{ + DerivationHash: eqClass.drvHashKey.toHash(), + OutputName: eqClass.outputName.Value(), + } +} + func (eqClass equivalenceClass) String() string { if eqClass.isZero() { return "ε" diff --git a/internal/backend/realize.go b/internal/backend/realize.go index 77f4948..de78472 100644 --- a/internal/backend/realize.go +++ b/internal/backend/realize.go @@ -819,13 +819,18 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets _, err = os.Lstat(b.server.realPath(outputPath)) log.Debugf(ctx, "%s exists=%t (output of %s)", outputPath, err == nil, drvPath) if err == nil { - outputs := map[string]realizationOutput{ - zbstore.DefaultDerivationOutputName: { - path: outputPath, - // Fixed outputs don't have references. + outputs := zbstore.RealizationMap{ + DerivationHash: drvHash, + Realizations: map[string][]*zbstore.Realization{ + zbstore.DefaultDerivationOutputName: { + { + OutputPath: outputPath, + // Fixed outputs don't have references. + }, + }, }, } - if err := b.recordRealizations(ctx, conn, drvHash, buildResultID, outputs); err != nil { + if err := b.recordRealizations(ctx, conn, buildResultID, outputs); err != nil { return fmt.Errorf("build %s: %v", drvPath, err) } return nil @@ -906,7 +911,10 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets return err } inputPaths := sets.CollectSorted(maps.Keys(inputs)) - outputs := make(map[string]realizationOutput) + outputs := zbstore.RealizationMap{ + DerivationHash: drvHash, + Realizations: make(map[string][]*zbstore.Realization), + } for outputName, tempOutputPath := range tempOutPaths { ref := zbstore.OutputReference{ DrvPath: drvPath, @@ -926,28 +934,35 @@ func (b *builder) do(ctx context.Context, drvPath zbstore.Path, outputNames sets drvPath, outputName, info.StorePath, prev.path) } - outputs[outputName] = realizationOutput{ - path: info.StorePath, - references: maps.Collect(func(yield func(zbstore.Path, sets.Set[equivalenceClass]) bool) { - for ref, eqClasses := range inputs { - if info.References.Has(ref) { - if !yield(ref, eqClasses) { - return - } - } + r := &zbstore.Realization{OutputPath: info.StorePath} + for ref, eqClasses := range inputs { + if info.References.Has(ref) { + for eqClass := range eqClasses.All() { + r.ReferenceClasses = append(r.ReferenceClasses, &zbstore.ReferenceClass{ + Path: ref, + Realization: zbstore.NonNull(eqClass.toRealizationOutputReference()), + }) } - }), + } + } + r.Signatures, err = b.server.keyring.Sign(zbstore.RealizationOutputReference{ + DerivationHash: drvHash, + OutputName: outputName, + }, r) + if err != nil { + log.Warnf(ctx, "Signing built realization: %v", err) } + outputs.Realizations[outputName] = []*zbstore.Realization{r} } // Record realizations. - if err := b.recordRealizations(ctx, conn, drvHash, buildResultID, outputs); err != nil { + if err := b.recordRealizations(ctx, conn, buildResultID, outputs); err != nil { return fmt.Errorf("build %s: %v", drvPath, err) } log.Infof(ctx, "Built %s: %s", drvPath, formatOutputPaths(maps.Collect(func(yield func(string, zbstore.Path) bool) { - for outputName, r := range outputs { - if !yield(outputName, r.path) { + for ref, r := range outputs.All() { + if !yield(ref.OutputName, r.OutputPath) { return } } @@ -1623,19 +1638,26 @@ func rewriteAtPath(path string, baseOffset int64, newDigest string, rewriters [] // recordRealizations calls [recordRealizations] and [recordBuildOutputs] in a transaction // and on success, saves the realizations into b.realizations. // The outputs must exist in the store. -func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drvHash nix.Hash, buildResultID int64, outputs map[string]realizationOutput) (err error) { +func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, buildResultID int64, outputs zbstore.RealizationMap) (err error) { + if log.IsEnabled(log.Debug) { + outputPaths := make(map[string]zbstore.Path) + for ref, r := range outputs.All() { + outputPaths[ref.OutputName] = r.OutputPath + } + log.Debugf(ctx, "Recording realizations for %v: %s", outputs.DerivationHash, formatOutputPaths(outputPaths)) + } + endFn, err := sqlitex.ImmediateTransaction(conn) if err != nil { - return fmt.Errorf("record realizations for %v: %v", drvHash, err) + return fmt.Errorf("record realizations for %v: %v", outputs.DerivationHash, err) } defer endFn(&err) - - if err := recordRealizations(ctx, conn, b.server.keyring, drvHash, outputs); err != nil { + if err := recordRealizations(conn, outputs.All()); err != nil { return err } buildOutputs := func(yield func(string, zbstore.Path) bool) { - for outputName, output := range outputs { - if !yield(outputName, output.path) { + for ref, r := range outputs.All() { + if !yield(ref.OutputName, r.OutputPath) { return } } @@ -1644,10 +1666,10 @@ func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drv return err } - for outputName, output := range outputs { - eqClass := newEquivalenceClass(drvHash, outputName) + for ref, r := range outputs.All() { closure := make(map[zbstore.Path]sets.Set[equivalenceClass]) - pe := pathAndEquivalenceClass{path: output.path, equivalenceClass: eqClass} + eqClass := realizationOutputReferenceKey(ref) + pe := pathAndEquivalenceClass{path: r.OutputPath, equivalenceClass: eqClass} err := closurePaths(conn, pe, func(pe pathAndEquivalenceClass) bool { addToMultiMap(closure, pe.path, pe.equivalenceClass) return true @@ -1656,7 +1678,7 @@ func (b *builder) recordRealizations(ctx context.Context, conn *sqlite.Conn, drv return err } b.realizations[eqClass] = cachedRealization{ - path: output.path, + path: r.OutputPath, closure: closure, } }