11use crate :: config;
2- use crate :: proc;
32use crate :: diffoscope:: diffoscope;
43use crate :: download:: download;
4+ use crate :: proc;
55use rebuilderd_common:: Distro ;
66use rebuilderd_common:: api:: { Rebuild , BuildStatus } ;
77use rebuilderd_common:: errors:: * ;
88use rebuilderd_common:: errors:: { Context as _} ;
99use std:: borrow:: Cow ;
1010use std:: collections:: HashMap ;
11+ use std:: fs;
12+ use std:: io:: ErrorKind ;
1113use std:: path:: { Path , PathBuf } ;
1214use std:: time:: Duration ;
15+ use tokio:: fs:: File ;
16+ use tokio:: io:: AsyncReadExt ;
1317
1418pub struct Context < ' a > {
1519 pub distro : & ' a Distro ,
@@ -40,44 +44,113 @@ fn locate_script(distro: &Distro, script_location: Option<PathBuf>) -> Result<Pa
4044 bail ! ( "Failed to find a rebuilder backend" )
4145}
4246
47+ fn path_to_string ( path : & Path ) -> Result < String > {
48+ let s = path. to_str ( )
49+ . with_context ( || anyhow ! ( "Path contains invalid characters: {:?}" , path) ) ?;
50+ Ok ( s. to_string ( ) )
51+ }
52+
53+ pub async fn compare_files ( a : & Path , b : & Path ) -> Result < bool > {
54+ let mut buf1 = [ 0u8 ; 4096 ] ;
55+ let mut buf2 = [ 0u8 ; 4096 ] ;
56+
57+ info ! ( "Comparing {:?} with {:?}" , a, b) ;
58+ let mut f1 = File :: open ( a) . await
59+ . with_context ( || anyhow ! ( "Failed to open {:?}" , a) ) ?;
60+ let mut f2 = File :: open ( b) . await
61+ . with_context ( || anyhow ! ( "Failed to open {:?}" , b) ) ?;
62+
63+ let mut pos = 0 ;
64+ loop {
65+ // read up to 4k bytes from the first file
66+ let n = f1. read_buf ( & mut & mut buf1[ ..] ) . await ?;
67+
68+ // check if the first file is end-of-file
69+ if n == 0 {
70+ debug ! ( "First file is at end-of-file" ) ;
71+
72+ // check if other file is eof too
73+ let n = f2. read_buf ( & mut & mut buf2[ ..] ) . await ?;
74+ if n > 0 {
75+ info ! ( "Files are not identical, {:?} is longer" , b) ;
76+ return Ok ( false ) ;
77+ } else {
78+ return Ok ( true ) ;
79+ }
80+ }
81+
82+ // check the same chunk in the other file
83+ match f2. read_exact ( & mut buf2[ ..n] ) . await {
84+ Ok ( n) => n,
85+ Err ( err) if err. kind ( ) == ErrorKind :: UnexpectedEof => {
86+ info ! ( "Files are not identical, {:?} is shorter" , b) ;
87+ return Ok ( false ) ;
88+ } ,
89+ err => err?,
90+ } ;
91+
92+ if buf1[ ..n] != buf2[ ..n] {
93+ // get the exact position
94+ // this can't panic because we've already checked the slices are not equal
95+ let pos = pos + buf1[ ..n] . iter ( ) . zip (
96+ buf2[ ..n] . iter ( )
97+ ) . position ( |( a, b) |a != b) . unwrap ( ) ;
98+ info ! ( "Files {:?} and {:?} differ at position {}" , a, b, pos) ;
99+
100+ return Ok ( false ) ;
101+ }
102+
103+ // advance the number of bytes that are equal
104+ pos += n;
105+ }
106+ }
107+
43108pub async fn rebuild ( ctx : & Context < ' _ > , url : & str ) -> Result < Rebuild > {
109+ // setup
44110 let tmp = tempfile:: Builder :: new ( ) . prefix ( "rebuilderd" ) . tempdir ( ) ?;
45111
46- let ( input, filename) = download ( url, & tmp)
112+ let inputs_dir = tmp. path ( ) . join ( "inputs" ) ;
113+ fs:: create_dir ( & inputs_dir)
114+ . context ( "Failed to create inputs/ temp dir" ) ?;
115+
116+ let out_dir = tmp. path ( ) . join ( "out" ) ;
117+ fs:: create_dir ( & out_dir)
118+ . context ( "Failed to create out/ temp dir" ) ?;
119+
120+ // download
121+ let filename = download ( url, & inputs_dir)
47122 . await
48123 . with_context ( || anyhow ! ( "Failed to download original package from {:?}" , url) ) ?;
49124
50- let ( success, log) = verify ( ctx, & input) . await ?;
51-
52- if success {
53- info ! ( "Rebuilder backend indicated a success rebuild!" ) ;
125+ // rebuild
126+ let input_path = inputs_dir. join ( & filename) ;
127+ let log = verify ( ctx, & out_dir, & input_path) . await ?;
128+
129+ // process result
130+ let output_path = out_dir. join ( & filename) ;
131+ if !output_path. exists ( ) {
132+ info ! ( "Build failed, no output artifact found at {:?}" , output_path) ;
133+ Ok ( Rebuild :: new ( BuildStatus :: Bad , log) )
134+ } else if compare_files ( & input_path, & output_path) . await ? {
135+ info ! ( "Files are identical, marking as GOOD" ) ;
54136 Ok ( Rebuild :: new ( BuildStatus :: Good , log) )
55137 } else {
56- info ! ( "Rebuilder backend exited with non-zero exit code " ) ;
138+ info ! ( "Build successful but artifacts differ " ) ;
57139 let mut res = Rebuild :: new ( BuildStatus :: Bad , log) ;
58140
59141 // generate diffoscope diff if enabled
60142 if ctx. diffoscope . enabled {
61- let output = Path :: new ( "./build/" ) . join ( filename) ;
62- if output. exists ( ) {
63- let output = output. to_str ( )
64- . ok_or_else ( || format_err ! ( "Output path contains invalid characters" ) ) ?;
65-
66- let diff = diffoscope ( & input, output, & ctx. diffoscope )
67- . await
68- . context ( "Failed to run diffoscope" ) ?;
69- res. diffoscope = Some ( diff) ;
70- } else {
71- info ! ( "Skipping diffoscope because rebuilder script did not produce output" ) ;
72- }
143+ let diff = diffoscope ( & input_path, & output_path, & ctx. diffoscope )
144+ . await
145+ . context ( "Failed to run diffoscope" ) ?;
146+ res. diffoscope = Some ( diff) ;
73147 }
74148
75149 Ok ( res)
76150 }
77151}
78152
79- // TODO: automatically truncate logs to a max-length if configured
80- async fn verify ( ctx : & Context < ' _ > , path : & str ) -> Result < ( bool , String ) > {
153+ async fn verify ( ctx : & Context < ' _ > , out_dir : & Path , input_path : & Path ) -> Result < String > {
81154 let bin = if let Some ( script) = ctx. script_location {
82155 Cow :: Borrowed ( script)
83156 } else {
@@ -86,15 +159,10 @@ async fn verify(ctx: &Context<'_>, path: &str) -> Result<(bool, String)> {
86159 Cow :: Owned ( script)
87160 } ;
88161
89- // TODO: establish a common interface to interface with distro rebuilders
90- // TODO: specify the path twice because the 2nd argument used to be the path
91- // TODO: we want to move this to the first instead. the 2nd argument can be removed in the future
92- let args = & [ path, path] ;
93-
94162 let timeout = ctx. build . timeout . unwrap_or ( 3600 * 24 ) ; // 24h
95163
96164 let mut envs = HashMap :: new ( ) ;
97- envs. insert ( "REBUILDERD_OUTDIR" . into ( ) , "./build" . into ( ) ) ;
165+ envs. insert ( "REBUILDERD_OUTDIR" . into ( ) , path_to_string ( out_dir ) ? ) ;
98166
99167 let opts = proc:: Options {
100168 timeout : Duration :: from_secs ( timeout) ,
@@ -103,5 +171,48 @@ async fn verify(ctx: &Context<'_>, path: &str) -> Result<(bool, String)> {
103171 passthrough : !ctx. build . silent ,
104172 envs,
105173 } ;
106- proc:: run ( bin. as_ref ( ) , args, opts) . await
174+ let ( _success, log) = proc:: run ( bin. as_ref ( ) , & [ input_path] , opts) . await ?;
175+
176+ Ok ( log)
177+ }
178+
179+ #[ cfg( test) ]
180+ mod tests {
181+ use super :: * ;
182+
183+ #[ tokio:: test]
184+ async fn compare_files_equal ( ) {
185+ let equal = compare_files ( Path :: new ( "src/main.rs" ) , Path :: new ( "src/main.rs" ) ) . await . unwrap ( ) ;
186+ assert ! ( equal) ;
187+ }
188+
189+ #[ tokio:: test]
190+ async fn compare_files_not_equal1 ( ) {
191+ let equal = compare_files ( Path :: new ( "src/main.rs" ) , Path :: new ( "Cargo.toml" ) ) . await . unwrap ( ) ;
192+ assert ! ( !equal) ;
193+ }
194+
195+ #[ tokio:: test]
196+ async fn compare_files_not_equal2 ( ) {
197+ let equal = compare_files ( Path :: new ( "Cargo.toml" ) , Path :: new ( "src/main.rs" ) ) . await . unwrap ( ) ;
198+ assert ! ( !equal) ;
199+ }
200+
201+ #[ tokio:: test]
202+ async fn compare_large_files_equal ( ) {
203+ let dir = tempfile:: tempdir ( ) . unwrap ( ) ;
204+ fs:: write ( dir. path ( ) . join ( "a" ) , & [ 0u8 ; 4096 * 100 ] ) . unwrap ( ) ;
205+ fs:: write ( dir. path ( ) . join ( "b" ) , & [ 0u8 ; 4096 * 100 ] ) . unwrap ( ) ;
206+ let equal = compare_files ( & dir. path ( ) . join ( "a" ) , & dir. path ( ) . join ( "b" ) ) . await . unwrap ( ) ;
207+ assert ! ( equal) ;
208+ }
209+
210+ #[ tokio:: test]
211+ async fn compare_large_files_not_equal ( ) {
212+ let dir = tempfile:: tempdir ( ) . unwrap ( ) ;
213+ fs:: write ( dir. path ( ) . join ( "a" ) , & [ 0u8 ; 4096 * 100 ] ) . unwrap ( ) ;
214+ fs:: write ( dir. path ( ) . join ( "b" ) , & [ 1u8 ; 4096 * 100 ] ) . unwrap ( ) ;
215+ let equal = compare_files ( & dir. path ( ) . join ( "a" ) , & dir. path ( ) . join ( "b" ) ) . await . unwrap ( ) ;
216+ assert ! ( !equal) ;
217+ }
107218}
0 commit comments