Merge pull request #88 from kpcyrd/new-sync

Prepare building once per pkgbase

Author: kpcyrd

Cargo.lock

@@ -12,11 +12,12 @@ RUN --mount=type=cache,target=/var/cache/buildkit \
         /var/cache/buildkit/target/release/rebuildctl /
 
 FROM alpine:3.14
-ENV HTTP_ADDR=0.0.0.0:8484
 RUN apk add --no-cache libgcc openssl dpkg sqlite-libs xz zstd-libs
 COPY --from=0 \
     /rebuilderd /rebuildctl \
     /usr/local/bin/
+ENV HTTP_ADDR=0.0.0.0:8484
+ENV REBUILDERD_COOKIE_PATH=/secret/auth
 VOLUME ["/data", "/secret"]
 WORKDIR /data
 CMD ["rebuilderd"]

Dockerfile

@@ -54,7 +54,6 @@ package_rebuilderd() {
   install -Dm 644 contrib/systemd/rebuilderd.tmpfiles "${pkgdir}/usr/lib/tmpfiles.d/rebuilderd.conf"
 
   # install docs
-  install -Dm 644 README.md -t "${pkgdir}/usr/share/doc/${pkgname}"
   install -Dm 644 -t "${pkgdir}/usr/share/man/man1" \
     contrib/docs/rebuilderd.1 \
     contrib/docs/rebuilderd-worker.1

PKGBUILD

@@ -13,8 +13,6 @@ edition = "2018"
 anyhow = "1.0.32"
 log = "0.4"
 serde = { version="1.0", features=["derive"] }
-strum = "0.21"
-strum_macros = "0.21"
 reqwest = { version="0.11", features=["blocking", "json"] }
 chrono = { version = "0.4", features=["serde"] }
 colored = "2"

common/Cargo.toml

@@ -1,7 +1,7 @@
 use chrono::prelude::*;
 use crate::config::ConfigFile;
 use crate::errors::*;
-use crate::{PkgRelease, PkgGroup, Status};
+use crate::{PkgRelease, PkgArtifact, PkgGroup, Status};
 use crate::auth;
 use reqwest::{Client as HttpClient, RequestBuilder};
 use serde::{Serialize, Deserialize};
@@ -300,7 +300,7 @@ pub enum JobAssignment {
 pub struct SuiteImport {
     pub distro: String,
     pub suite: String,
-    pub pkgs: Vec<PkgGroup>,
+    pub groups: Vec<PkgGroup>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]
@@ -321,7 +321,7 @@ pub struct QueueList {
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct QueueItem {
     pub id: i32,
-    pub package: PkgRelease,
+    pub pkgbase: PkgGroup,
     pub version: String,
     pub queued_at: NaiveDateTime,
     pub worker_id: Option<i32>,
@@ -393,7 +393,7 @@ impl Rebuild {
 #[derive(Debug, Serialize, Deserialize)]
 pub struct BuildReport {
     pub queue: QueueItem,
-    pub rebuild: Rebuild,
+    pub rebuilds: Vec<(PkgArtifact, Rebuild)>,
 }
 
 #[derive(Debug, Serialize, Deserialize)]

common/src/api.rs

@@ -1,7 +1,6 @@
 use crate::errors::*;
 use colored::*;
 use chrono::NaiveDateTime;
-use strum_macros::{EnumString, AsRefStr, Display};
 use serde::{Serialize, Deserialize};
 use std::iter::FromIterator;
 use std::ops::Deref;
@@ -13,23 +12,6 @@ pub mod config;
 pub mod errors;
 pub mod utils;
 
-#[derive(Debug, Clone, Copy, PartialEq, Display, EnumString, AsRefStr, Serialize, Deserialize)]
-#[strum(serialize_all = "kebab-case")]
-#[serde(rename_all = "kebab-case")]
-pub enum VersionCmp {
-    Basic,
-    Debian,
-}
-
-impl VersionCmp {
-    pub fn detect_from_distro(distro: &str) -> VersionCmp {
-        match distro {
-            "debian" => VersionCmp::Debian,
-            _ => VersionCmp::Basic,
-        }
-    }
-}
-
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 pub struct PkgRelease {
     pub name: String,
@@ -39,16 +21,14 @@ pub struct PkgRelease {
     pub suite: String,
     pub architecture: String,
     pub artifact_url: String,
-    pub input_url: Option<String>,
     pub build_id: Option<i32>,
     pub built_at: Option<NaiveDateTime>,
     pub has_diffoscope: bool,
     pub has_attestation: bool,
-    pub next_retry: Option<NaiveDateTime>,
 }
 
 impl PkgRelease {
-    pub fn new(name: String, version: String, distro: String, suite: String, architecture: String, artifact_url: String, input_url: Option<String>) -> PkgRelease {
+    pub fn new(name: String, version: String, distro: String, suite: String, architecture: String, artifact_url: String) -> PkgRelease {
         PkgRelease {
             name,
             version,
@@ -57,45 +37,43 @@ impl PkgRelease {
             suite,
             architecture,
             artifact_url,
-            input_url,
             build_id: None,
             built_at: None,
             has_diffoscope: false,
             has_attestation: false,
-            next_retry: None,
         }
     }
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub struct PkgGroup {
-    pub base: String,
+    pub name: String,
     pub version: String,
 
     pub distro: String,
     pub suite: String,
     pub architecture: String,
 
-    pub input: Option<String>,
+    pub input_url: Option<String>,
     pub artifacts: Vec<PkgArtifact>,
 }
 
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]
 pub struct PkgArtifact {
     pub name: String,
     pub version: String,
     pub url: String,
 }
 
 impl PkgGroup {
-    pub fn new(base: String, version: String, distro: String, suite: String, architecture: String, input: Option<String>) -> PkgGroup {
+    pub fn new(name: String, version: String, distro: String, suite: String, architecture: String, input_url: Option<String>) -> PkgGroup {
         PkgGroup {
-            base,
+            name,
             version,
             distro,
             suite,
             architecture,
-            input,
+            input_url,
             artifacts: Vec::new(),
         }
     }
@@ -104,9 +82,9 @@ impl PkgGroup {
         self.artifacts.push(artifact);
     }
 
-    pub fn input(&self) -> Result<&str> {
-        if let Some(input) = &self.input {
-            Ok(input.as_str())
+    pub fn input_url(&self) -> Result<&str> {
+        if let Some(input_url) = &self.input_url {
+            Ok(input_url.as_str())
         } else if !self.artifacts.is_empty() {
             let mut artifacts = Vec::from_iter(self.artifacts.iter().collect::<Vec<_>>());
             artifacts.sort_by_key(|a| &a.name);

common/src/lib.rs

@@ -26,9 +26,11 @@ diesel_migrations = { version = "1.3.0", features = ["sqlite"] }
 dirs-next = "2"
 dotenv = "0.15"
 env_logger = "0.9"
+log = "0.4.14"
 rand = "0.8"
 rebuilderd-common = { version= "=0.15.0", path="../common" }
 serde = { version="1.0", features=["derive"] }
+serde_json = "1.0.71"
 structopt = "0.3"
 toml = "0.5.6"
 

daemon/Cargo.toml

@@ -0,0 +1,41 @@
+DROP TABLE queue;
+
+CREATE TABLE queue (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    package_id INTEGER NOT NULL,
+    version VARCHAR NOT NULL,
+    required_backend VARCHAR NOT NULL,
+    priority INTEGER NOT NULL,
+    queued_at DATETIME NOT NULL,
+    worker_id INTEGER,
+    started_at DATETIME,
+    last_ping DATETIME,
+    FOREIGN KEY(package_id) REFERENCES packages(id) ON DELETE CASCADE,
+    FOREIGN KEY(worker_id) REFERENCES workers(id) ON DELETE SET NULL,
+    CONSTRAINT queue_unique UNIQUE (package_id, version)
+);
+
+CREATE UNIQUE INDEX queue_pop_idx ON queue(required_backend, priority, queued_at, id);
+
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _pkgbases_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    name VARCHAR NOT NULL,
+    version VARCHAR NOT NULL,
+    distro VARCHAR NOT NULL,
+    suite VARCHAR NOT NULL,
+    architecture VARCHAR NOT NULL,
+    retries INTEGER NOT NULL,
+    next_retry DATETIME,
+    CONSTRAINT pkgbase_unique UNIQUE (name, version, distro, suite, architecture)
+);
+
+INSERT INTO _pkgbases_new (id, name, version, distro, suite, architecture, retries, next_retry)
+    SELECT id, name, version, distro, suite, architecture, retries, next_retry
+    FROM pkgbases;
+
+DROP TABLE pkgbases;
+ALTER TABLE _pkgbases_new RENAME TO pkgbases;
+
+PRAGMA foreign_keys=on;

daemon/migrations/2021-11-21-134730_build-groups/down.sql

@@ -0,0 +1,43 @@
+DROP TABLE queue;
+
+CREATE TABLE queue (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    pkgbase_id INTEGER NOT NULL,
+    version VARCHAR NOT NULL,
+    required_backend VARCHAR NOT NULL,
+    priority INTEGER NOT NULL,
+    queued_at DATETIME NOT NULL,
+    worker_id INTEGER,
+    started_at DATETIME,
+    last_ping DATETIME,
+    FOREIGN KEY(pkgbase_id) REFERENCES pkgbases(id) ON DELETE CASCADE,
+    FOREIGN KEY(worker_id) REFERENCES workers(id) ON DELETE SET NULL,
+    CONSTRAINT queue_unique UNIQUE (pkgbase_id, version)
+);
+
+CREATE UNIQUE INDEX queue_pop_idx ON queue(required_backend, priority, queued_at, id);
+
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _pkgbases_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    name VARCHAR NOT NULL,
+    version VARCHAR NOT NULL,
+    distro VARCHAR NOT NULL,
+    suite VARCHAR NOT NULL,
+    architecture VARCHAR NOT NULL,
+    input_url VARCHAR,
+    artifacts VARCHAR NOT NULL,
+    retries INTEGER NOT NULL,
+    next_retry DATETIME,
+    CONSTRAINT pkgbase_unique UNIQUE (name, version, distro, suite, architecture)
+);
+
+INSERT INTO _pkgbases_new (id, name, version, distro, suite, architecture, artifacts, retries, next_retry)
+    SELECT id, name, version, distro, suite, architecture, "[]", retries, next_retry
+    FROM pkgbases;
+
+DROP TABLE pkgbases;
+ALTER TABLE _pkgbases_new RENAME TO pkgbases;
+
+PRAGMA foreign_keys=on;

daemon/migrations/2021-11-21-134730_build-groups/up.sql

@@ -0,0 +1,33 @@
+PRAGMA foreign_keys=off;
+
+CREATE TABLE _packages_new (
+    id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL,
+    base_id INTEGER,
+    name VARCHAR NOT NULL,
+    version VARCHAR NOT NULL,
+    status VARCHAR NOT NULL,
+    distro VARCHAR NOT NULL,
+    suite VARCHAR NOT NULL,
+    architecture VARCHAR NOT NULL,
+    artifact_url VARCHAR NOT NULL,
+    input_url VARCHAR,
+    build_id INTEGER,
+    built_at DATETIME,
+    has_diffoscope BOOLEAN NOT NULL,
+    has_attestation BOOLEAN NOT NULL,
+    checksum VARCHAR,
+    retries INTEGER NOT NULL,
+    next_retry DATETIME,
+    CONSTRAINT packages_unique UNIQUE (name, distro, suite, architecture),
+    FOREIGN KEY(base_id) REFERENCES pkgbases(id),
+    FOREIGN KEY(build_id) REFERENCES builds(id)
+);
+
+INSERT INTO _packages_new (id, base_id, name, version, status, distro, suite, architecture, artifact_url, build_id, built_at, has_diffoscope, has_attestation, checksum, retries)
+    SELECT id, pkgbase_id, name, version, status, distro, suite, architecture, artifact_url, build_id, built_at, has_diffoscope, has_attestation, checksum, 0
+    FROM packages;
+
+DROP TABLE packages;
+ALTER TABLE _packages_new RENAME TO packages;
+
+PRAGMA foreign_keys=on;