diff --git a/Cargo.toml b/Cargo.toml index 01a4085..391ec0b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,7 @@ flate2 = "1.0.13" handlebars = "3.0.1" attohttpc = "0.13" regex = "0.1.41" -rustc-serialize = "0.3.16" +rustc-serialize = "0.3.24" time = "0.1.33" toml = "0.1.23" walkdir = "=0.1.3" -term = "=0.2.12" diff --git a/src/cli.rs b/src/cli.rs index 5dbacbe..f5d035a 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -111,7 +111,7 @@ pub fn fetch_rng(from: Option, to: Option) -> () { } for date in date_strings { - fetch(date); + fetch(date) } } @@ -121,7 +121,13 @@ pub fn fetch(s: String) -> () { Some(a) => a, None => panic!("Could not fetch archive"), }; - archive.fetch(); + + print!("fetch {} ... ", archive); + + match archive.fetch() { + Ok(status) => println!("ok: {}", status), + Err(status) => println!("error: {}", status), + } } pub fn ls_data() -> () { @@ -154,11 +160,9 @@ pub fn ls_data() -> () { /// Print the standard paths that the app uses. pub fn show_paths() -> () { - let v = vec![config_path(), data_path(), config_file_path()]; - - v.into_iter() - .map(|e| ::print_green(format!(" {}\n", e.to_str().unwrap()).as_ref())) - .for_each(drop); + for e in vec![config_path(), data_path(), config_file_path()].iter() { + println!("{}", e.to_string_lossy()); + } } /// TODO: I'm not sure if this is supported anymore? diff --git a/src/config.rs b/src/config.rs index 58e5831..8463945 100644 --- a/src/config.rs +++ b/src/config.rs @@ -17,11 +17,9 @@ const CONFIG: &'static str = "gar.toml"; pub fn config_path() -> PathBuf { let home = match env::var("HOME") { Ok(v) => v, - Err(e) => { - println!("Could not get the HOME environment variable"); - panic!(e); - }, + Err(e) => panic!("could not get the HOME environment variable: {}", e), }; + let mut path = PathBuf::new(); path.push(home); path.push(PREFIX); @@ -44,7 +42,7 @@ pub fn config_file_path() -> PathBuf { fn read_configuration_file() -> Table { let mut f: File = match File::open(config_file_path()) { Ok(v) => v, - Err(e) => panic!(e), + Err(e) => panic!("{}", e), }; let mut s: String = String::new(); f.read_to_string(&mut s).unwrap(); @@ -84,16 +82,18 @@ pub fn data_exists(filename: &String) -> bool { /// Default things to run each time we go through the main entry point. pub fn init() -> () { let cpath: PathBuf = config_path(); + let cpath_str = cpath.to_string_lossy(); let dpath: PathBuf = data_path(); + let dpath_str = dpath.to_string_lossy(); let config_file: PathBuf = config_file_path(); if !cpath.exists() { - println!("Config file path created for the first time"); + println!("config file path created for the first time: {}", cpath_str); fs::create_dir_all(cpath.to_str().unwrap()).unwrap(); } if !dpath.exists() { - println!("Data path created for the first time"); + println!("data path created for the first time: {}", dpath_str); fs::create_dir_all(dpath.to_str().unwrap()).unwrap(); } @@ -108,12 +108,10 @@ pub fn init() -> () { let mut f: File = match File::create(config_file) { Ok(v) => v, - Err(e) => panic!("Could not open config file for writing {}", e), + Err(e) => panic!("could not open config file for writing {}", e), }; - println!("Writing configuration for the first time"); + println!("writing configuration for the first time"); f.write_all(s.as_bytes()).unwrap(); } } - - diff --git a/src/lib.rs b/src/lib.rs index 84003a0..952d527 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,9 @@ -//! This is a small tool that helps you interface with githubarchive.org -//! It provides utilities to fetch specific archives, or fetch a range of archives via date. -//! You can then run some semi-complex queries on the downloaded archives, which are still in gz -//! format. Try running `help` to see information of each command, and subcommands. +//! This is a small tool that helps you interface with +//! githubarchive.org It provides utilities to fetch specific +//! archives, or fetch a range of archives via date. You can then run +//! some semi-complex queries on the downloaded archives, which are +//! still in gz format. Try running `help` to see information of each +//! command, and subcommands. //! //! ````nocode //! gar 0.2.0 @@ -82,24 +84,32 @@ //! //! Here is an example of a query: //! -//! gar query --where language:Rust,type:create +//! ```nocode +//! gar query --where language:Rust,type:create +//! ``` //! //! This will search for all events, and select only the events where the repository is of the Rust //! language, and the type of event is a CreateEvent. You can also specify time constraints with to //! and from: //! -//! gar query --where language:Rust,type:create --from 2013-1-1-1 --to 2013-1-5-1 +//! ```nocode +//! gar query --where language:Rust,type:create --from 2013-1-1-1 --to 2013-1-5-1 +//! ``` //! //! And as you noticed you can also provide a type of event, and language using the `--where` clause. //! The way you do this, is by providing a label, delimited with a colon `:` and provide the value. //! For example: //! -//! language:Rust +//! ```nocode +//! language:Rust +//! ``` //! //! Satisfies this query. You can add more constraints by delimiting them with a comma ','. The //! relevance of a comma in this case is as if it's a logical `AND`. As you previously saw: //! -//! language:Rust,type:create +//! ```nocode +//! language:Rust,type:create +//! ``` //! //! Here's the list of things you can add as constraints: //! @@ -208,7 +218,6 @@ extern crate toml; extern crate chrono; extern crate attohttpc; extern crate regex; -extern crate term; extern crate flate2; extern crate walkdir; extern crate handlebars; @@ -216,63 +225,3 @@ extern crate handlebars; pub mod models; pub mod config; pub mod cli; - -#[inline] -fn print_yellow(s: &str) -> () { - generic_print(s, term::color::YELLOW); -} - -#[inline] -fn print_green(s: &str) -> () { - generic_print(s, term::color::GREEN); -} - -#[inline] -fn print_red(s: &str) -> () { - let mut t = term::stderr().unwrap(); - t.fg(term::color::RED).unwrap(); - write!(t, "{}", s).unwrap(); - t.reset().unwrap(); -} - -#[inline] -fn print_magenta(s: &str) -> () { - generic_print(s, term::color::MAGENTA); -} - -#[inline] -fn generic_print(s: &str, col: term::color::Color) -> () { - let mut t = term::stdout().unwrap(); - t.fg(col).unwrap(); - write!(t, "{}", s).unwrap(); - t.reset().unwrap(); -} - -fn vec_contains(v: &Vec, t: &T) -> bool { - for e in v { - if e == t { return true } - } - false -} - - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn test_vec_contains() -> () { - let vs: Vec = vec!["potato".into(), "yotato".into(), "motato".into()]; - let vu32: Vec = vec![1,2,3,4,5,6]; - let vi32: Vec = vec![1,2,3,4,5]; - let jon: String = "jon".into(); - - assert!(::vec_contains::(&vs, &"potato".into())); - assert!(::vec_contains::(&vu32, &2)); - assert!(::vec_contains::(&vi32, &5)); - - assert!(!::vec_contains::(&vi32, &122)); - assert!(!::vec_contains::(&vu32, &123123)); - assert!(!::vec_contains::(&vs, &jon)); - } -} diff --git a/src/models/archive.rs b/src/models/archive.rs index f325c9d..20ec084 100644 --- a/src/models/archive.rs +++ b/src/models/archive.rs @@ -1,16 +1,40 @@ use chrono::*; use std::fs::File; +use std::fmt; use std::path::PathBuf; use std::io::Write; use config; -const GITHUT_ARCHIVE_URL: &'static str = "https://data.githubarchive.org/"; +const GITHUT_ARCHIVE_URL: &str = "https://data.githubarchive.org/"; + +// TODO might be worth to break into FetchErrors and FetchOks +pub enum FetchStatus { + Success, + Cached, + FailFetch, + NotFound, + CantCreateCacheFile, + CantWriteCacheFile, + ResourceUnavailable, +} + +impl fmt::Display for FetchStatus { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + FetchStatus::Success => write!(f, "success"), + FetchStatus::Cached => write!(f, "cached"), + FetchStatus::FailFetch => write!(f, "failed to fetch"), + FetchStatus::NotFound => write!(f, "not found"), + FetchStatus::CantCreateCacheFile => write!(f, "cant create cache file"), + FetchStatus::CantWriteCacheFile => write!(f, "cant write cache file"), + FetchStatus::ResourceUnavailable => write!(f, "resource unavailable"), + } + } +} -/// An archive is a file object - not to be confused with repos, or things that will give us access -/// to data. #[derive(Debug)] pub struct Archive { date: DateTime, @@ -22,13 +46,12 @@ pub struct ArchiveBuilder { year: i32, month: u32, day: u32, - hour: u32 + hour: u32, } impl Archive { - - pub fn new(y: i32, m: u32, d: u32, h: u32) -> Archive { - let d = Utc.ymd(y, m, d).and_hms(h, 0, 0); + pub fn new(year: i32, month: u32, day: u32, hour: u32) -> Archive { + let d = Utc.ymd(year, month, day).and_hms(hour, 0, 0); let n = Archive::make_title(d); Archive { @@ -46,81 +69,88 @@ impl Archive { format!("{}.json.gz", Archive::make_date(d)) } - fn fetch_raw(url: &String) -> Vec { - match attohttpc::get(&url).send() { - Ok(raw) => raw.bytes().unwrap(), - Err(e) => panic!(e), + fn fetch_raw(url: &String) -> Result, FetchStatus> { + let response = match attohttpc::get(&url).send() { + Ok(v) => v, + Err(_) => return Err(FetchStatus::FailFetch), + }; + + if !response.is_success() { + // TODO: may be more cases here, but this will have to do + // for now. Apparently attohttpc actually uses hyperium, + // or some parts of it + return Err(FetchStatus::NotFound); + } + + match response.bytes() { + Ok(bytes) => Ok(bytes), + Err(_) => Err(FetchStatus::ResourceUnavailable), } } - /// Fetch the information of a specific archive. This will return something in memory, and will - /// not make a local copy. - pub fn fetch(&mut self) -> () { + pub fn fetch(&mut self) -> Result { let title: String = Archive::make_title(self.date); if config::data_exists(&title) { - ::print_yellow(format!("Data {} exists in cache - skip\n", title).as_ref()); - return; + return Ok(FetchStatus::Cached) } let url: String = format!("{}{}", GITHUT_ARCHIVE_URL, title); - self.data = Archive::fetch_raw(&url); - - if &self.data[0..5] == b" self.data = value, + Err(_) => return Err(FetchStatus::FailFetch), } - if config::caching_on() { self.store() } + if config::caching_on() { return self.store() } + + Ok(FetchStatus::Success) + } + + pub fn get_name(&self) -> String { + self.name.clone() } - pub fn store(&self) -> () { + pub fn store(&self) -> Result { let mut base: PathBuf = config::data_path(); - let s: String = match base.clone().to_str() { - Some(v) => v.to_string(), - None => return, - }; base.push(self.name.clone()); let mut f: File = match File::create(base) { Ok(v) => v, - Err(e) => { - println!("Problem opening caching file @ {:?}", s); - println!("{}", e); - return; - }, + Err(_) => return Err(FetchStatus::CantCreateCacheFile), }; - f.write_all(&self.data).unwrap(); + match f.write_all(&self.data) { + Ok(_) => Ok(FetchStatus::Success), + Err(_) => Err(FetchStatus::CantWriteCacheFile), + } } - /// Set the year of the archive we're interested in pub fn set_year(&mut self, year: i32) -> () { self.date = Utc.ymd(year, self.date.month(), self.date.day()) .and_hms(9, 0, 0); } - /// Set the month of the archive we're interested in pub fn set_month(&mut self, month: u32) -> () { self.date = Utc.ymd(self.date.year(), month, self.date.day()).and_hms(9, 0, 0); } - /// Set the day of the archive we're interested in pub fn set_day(&mut self, day: u32) -> () { self.date = Utc.ymd(self.date.year(), self.date.month(), day).and_hms(9, 0, 0); } - /// Set the hour of the archive we're interested in pub fn set_hour(&mut self, h: u32) -> () { self.date = Utc.ymd(self.date.year(), self.date.month(), self.date.day()).and_hms(h, 0, 0); } } +impl fmt::Display for Archive { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", Archive::make_date(self.date)) + } +} + impl ArchiveBuilder { pub fn new() -> ArchiveBuilder { ArchiveBuilder { diff --git a/src/models/event.rs b/src/models/event.rs index 91903f8..8c5cd04 100644 --- a/src/models/event.rs +++ b/src/models/event.rs @@ -1,3 +1,4 @@ +use std::io::ErrorKind; use std::collections::BTreeMap; use std::path::PathBuf; @@ -14,11 +15,12 @@ use models::payloads::*; use chrono::*; +// TODO: this should have an enumerated struct, because not all the +// events have the same fields. #[derive(Debug)] pub struct Event { gh_id: u64, name: String, - description: String, language: String, has_issues: bool, owner: owner::Owner, @@ -31,17 +33,15 @@ pub struct Event { created_at: Option>, } -/// Models a repo event, in the file obtained from githubarchive. -impl Event { - pub fn new() -> Event { - Event { +impl Default for Event { + fn default() -> Self { + Event{ gh_id: 0, - name: "default".into(), - language: "default".into(), - description: "default".into(), + name: "".into(), + language: "".into(), has_issues: false, owner: owner::OwnerBuilder::new().finalize(), - url: "default".into(), + url: "".into(), watchers: 0, stargazers: 0, forks: 0, @@ -50,6 +50,15 @@ impl Event { created_at: None, } } +} + +/// Models a repo event, in the file obtained from githubarchive. +impl Event { + pub fn new() -> Event { Default::default() } + + pub fn get_gh_id(&self) -> u64 { self.gh_id } + pub fn get_name(&self) -> &String { &self.name } + pub fn get_language(&self) -> &String { &self.language } pub fn set_owner_gh_id(&mut self, id: u64) -> () { self.owner.set_gh_id(id); @@ -87,12 +96,7 @@ impl Event { let re: Regex = Regex::new(re_str.as_ref()).unwrap(); b &= re.is_match(self.name.as_ref()); } - if cons.label == "description" { - /* This does a wor dmatch against the description given to the event's repo */ - let re_str: String = format!("(?i){}", cons.value); - let re: Regex = Regex::new(re_str.as_ref()).unwrap(); - b &= re.is_match(self.description.as_ref()); - } + // TODO: maybe make the payload description searchable if cons.label == "+watchers" { /* TODO: parsing to int each time - this might not be good? */ let num: u64 = cons.value.parse::().ok().unwrap(); @@ -119,6 +123,9 @@ impl Event { None => continue, }; + // TODO: there should bea better way to do this -- + // probably can convert the constraint into the enum + // and make the comparison like this b &= match cons.value.as_ref() { "create" => match etype { &EventType::Create => true, _ => false }, "commit_comment" => match etype { &EventType::CommitComment => true, _ => false }, @@ -173,38 +180,60 @@ impl Event { b } - /// Given a path to a json.gz file, that file is read, and each line is parsed to a Event - /// object. + /// Given a path to a json.gz file, that file is read, and each + /// line is parsed to a Event object. pub fn from_path(p: PathBuf) -> Vec { let v: Vec = lines_of(p); let mut res: Vec = Vec::new(); + for line in v.into_iter() { - let json_line: Json = match Json::from_str(line.as_ref()) { - Ok(v) => v, - Err(e) => { - ::print_red(format!("Could not parse anything given:\n{}", line).as_ref()); - ::print_red(format!("Err: {}", e).as_ref()); - continue; - }, + let obj = match Event::into_json_object(&line) { + Ok(v) => v, + Err(..) => continue, // TODO: logging should be good here }; - if let Some(v) = Event::from_json(Some(&json_line)) { - res.push(v); - }; + res.push(obj); } res } + pub fn into_json_object(line: &str) -> std::io::Result { + // cleanup stdio::Error + + let json_line: Json = match Json::from_str(line) { + Ok(v) => v, + Err(e) => { + // TODO cleanup + println!("Could not parse anything given:\n{}", line); + println!("Err: {}", e); + return Err(std::io::Error::new(ErrorKind::InvalidInput, "invalid json")) + }, + }; + + match Event::from_json(&json_line) { + Some(v) => Ok(v), + None => { + return Err(std::io::Error::new(ErrorKind::InvalidInput, "bad json structure")) + }, + } + } + /// Given a json string, try to evaluate it into a repo - pub fn from_json(json: Option<&Json>) -> Option { - if json.is_none() { return None } - if !json.unwrap().is_object() { return None } + pub fn from_json(json: &Json) -> Option { + const KEY_CREATED_AT: &str = "created_at"; + const KEY_REPOSITORY: &str = "repo"; + const KEY_TYPE: &str = "type"; + // I'm adding the above as constants; some of the keys changed + // in the json over the years, and we should have one source of + // truth. + + if !json.is_object() { return None } - let obj = json.unwrap().as_object().unwrap(); + let obj = json.as_object().unwrap(); - let created_at: Option> = match obj.get("created_at") { + let created_at: Option> = match obj.get(KEY_CREATED_AT) { Some(v) => { match *v { Json::String(ref s) => { @@ -220,52 +249,35 @@ impl Event { None => None }; - let repo = match obj.get("repository") { - None => return None, - Some(v) => v.as_object().unwrap(), - }; + let payload_obj = obj.get("payload"); - let event: Option = match obj.get("type") { + let event: Option = match obj.get(KEY_TYPE) { None => None, Some(v) => match *v { Json::String(ref s) => { - match s.as_ref() { - "CreateEvent" => Some(EventType::Create), - "CommitCommentEvent" => Some(EventType::CommitComment), - "DeleteEvent" => Some(EventType::Delete(DeletePayload::from_json(obj.get("payload")))), - "DeploymentEvent" => Some(EventType::Deployment), - "DeploymentStatusEvent" => Some(EventType::DeploymentStatus), - "DownloadEvent" => Some(EventType::Download), - "FollowEvent" => Some(EventType::Follow), - "ForkEvent" => Some(EventType::Fork), - "ForkApplyEvent" => Some(EventType::ForkApply), - "GistEvent" => Some(EventType::Gist), - "GollumEvent" => Some(EventType::Gollum(GollumPayload::from_json(obj.get("payload")))), - "IssueCommentEvent" => Some(EventType::IssueComment(IssueCommentPayload::from_json(obj.get("payload")))), - "IssuesEvent" => Some(EventType::Issues(IssuePayload::from_json(obj.get("payload")))), - "MemberEvent" => Some(EventType::Member), - "MembershipEvent" => Some(EventType::Membership), - "PageBuildEvent" => Some(EventType::PageBuild), - "PublicEvent" => Some(EventType::Public), - "PullRequestEvent" => Some(EventType::PullRequest), - "PullRequestReviewCommentEvent" => Some(EventType::PullRequestReviewComment), - "PushEvent" => Some(EventType::Push(PushPayload::from_json(obj.get("payload")))), - "ReleaseEvent" => Some(EventType::Release), - "RepositoryEvent" => Some(EventType::Repository), - "StatusEvent" => Some(EventType::Status), - "TeamAddEvent" => Some(EventType::TeamAdd), - "WatchEvent" => Some(EventType::Watch(WatchPayload::from_json(obj.get("payload")))), - _ => None, + match s.parse::().unwrap() { + EventType::Unknown(..) => None, // preserve Unknown in the future + EventType::Delete(..) => Some(EventType::Delete(DeletePayload::from_json(payload_obj))), + EventType::Gollum(..) => Some(EventType::Gollum(GollumPayload::from_json(payload_obj))), + EventType::IssueComment(..) => Some(EventType::IssueComment(IssueCommentPayload::from_json(payload_obj))), + EventType::Issues(..) => Some(EventType::Issues(IssuePayload::from_json(payload_obj))), + EventType::Push(..) => Some(EventType::Push(PushPayload::from_json(payload_obj))), + EventType::Watch(..) => Some(EventType::Watch(WatchPayload::from_json(payload_obj))), + simple_event => Some(simple_event), } }, _ => None, }, }; + let repo = match obj.get(KEY_REPOSITORY) { + None => return None, + Some(v) => v.as_object().unwrap(), + }; + let gh_id = JsonHelper::number_or_zero(repo.get("id")); let name: String = JsonHelper::string_or_empty(repo.get("name")); let url: String = JsonHelper::string_or_empty(repo.get("url")); - let desc: String = JsonHelper::string_or_empty(repo.get("description")); let owner_name: String = JsonHelper::string_or_empty(repo.get("owner")); let issues_present: bool = JsonHelper::boolean_or_false(repo.get("has_issues")); let language: String = JsonHelper::string_or_empty(repo.get("language")); @@ -280,7 +292,6 @@ impl Event { repo.gh_id = gh_id; repo.url = url; - repo.description = desc; repo.name = name; repo.has_issues = issues_present; repo.language = language; @@ -294,83 +305,11 @@ impl Event { Some(repo) } - /// Gives a flat json hash with labels and values. - pub fn to_btree_with_features_of(&self, f: Vec) -> BTreeMap { - let mut map: BTreeMap = BTreeMap::new(); - - let id_label: String = "id".into(); - let name_label: String = "name".into(); - let desc: String = "description".into(); - let lang: String = "language".into(); - let has_issues: String = "has_issues".into(); - let owner: String = "owner".into(); - let url: String = "url".into(); - let watchers: String = "watchers".into(); - let stargazers: String = "stargazers".into(); - let forks: String = "forks".into(); - let open_issues: String = "forks".into(); - let event_type: String = "event_type".into(); - let created_at: String = "created_at".into(); - - if ::vec_contains(&f, &id_label) { - map.insert(id_label, self.gh_id.to_string()); - } - if ::vec_contains(&f, &name_label) { - map.insert(name_label, self.name.clone()); - } - if ::vec_contains(&f, &desc) { - map.insert(desc, self.description.clone()); - } - if ::vec_contains(&f, &lang) { - map.insert(lang, self.language.clone()); - } - if ::vec_contains(&f, &has_issues) { - map.insert(has_issues, self.has_issues.to_string()); - } - if ::vec_contains(&f, &owner) { - map.insert(owner, self.owner.get_nick().clone()); - } - if ::vec_contains(&f, &url) { - map.insert(url, self.url.clone()); - } - if ::vec_contains(&f, &watchers) { - map.insert(watchers, self.watchers.to_string()); - } - if ::vec_contains(&f, &stargazers) { - map.insert(stargazers, self.stargazers.to_string()); - } - if ::vec_contains(&f, &forks) { - map.insert(forks, self.forks.to_string()); - } - if ::vec_contains(&f, &open_issues) { - map.insert(open_issues, self.open_issues.to_string()); - } - if ::vec_contains(&f, &event_type) { - if let Some(ref etype) = self.event_type { - map.insert(event_type, etype.to_string()); - } - else { - map.insert(event_type, "null".into()); - } - } - if ::vec_contains(&f, &created_at) { - if let Some(date) = self.created_at { - map.insert(created_at, date.to_rfc3339()); - } - else { - map.insert(created_at, "null".into()); - } - } - - map - } - pub fn to_btree_with_all_features(&self) -> BTreeMap { let mut map: BTreeMap = BTreeMap::new(); map.insert("id".into(), self.gh_id.to_string()); map.insert("name".into(), self.name.clone()); - map.insert("description".into(), self.description.clone()); map.insert("language".into(), self.language.clone()); map.insert("has_issues".into(), self.has_issues.to_string()); map.insert("owner".into(), self.owner.get_nick().clone()); diff --git a/src/models/event_type.rs b/src/models/event_type.rs index ab82f97..f514923 100644 --- a/src/models/event_type.rs +++ b/src/models/event_type.rs @@ -1,6 +1,8 @@ +use std::str::FromStr; + use models::payloads::*; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub enum EventType { Create, Fork, @@ -28,6 +30,7 @@ pub enum EventType { Status, TeamAdd, Watch(Option), + Unknown(String), } impl ToString for EventType { @@ -59,6 +62,44 @@ impl ToString for EventType { EventType::Status => "StatusEvent".into(), EventType::TeamAdd => "TeamAddEvent".into(), EventType::Watch(..) => "WatchEvent".into(), + EventType::Unknown(ref val) => format!("Unknown({})", val), } } } + +impl FromStr for EventType { + type Err = std::convert::Infallible; + + fn from_str(s: &str) -> Result { + let ret = match s { + "CreateEvent" => EventType::Create, + "CommitCommentEvent" => EventType::CommitComment, + "DeleteEvent" => EventType::Delete(None), + "DeploymentEvent" => EventType::Deployment, + "DeploymentStatusEvent" => EventType::DeploymentStatus, + "DownloadEvent" => EventType::Download, + "FollowEvent" => EventType::Follow, + "ForkEvent" => EventType::Fork, + "ForkApplyEvent" => EventType::ForkApply, + "GistEvent" => EventType::Gist, + "GollumEvent" => EventType::Gollum(None), + "IssueCommentEvent" => EventType::IssueComment(None), + "IssuesEvent" => EventType::Issues(None), + "MemberEvent" => EventType::Member, + "MemebershipEvent" => EventType::Membership, + "PageBuildEvent" => EventType::PageBuild, + "PublicEvent" => EventType::Public, + "PullRequestEvent" => EventType::PullRequest, + "PullRequestReviewCommentEvent" => EventType::PullRequestReviewComment, + "PushEvent" => EventType::Push(None), + "ReleaseEvent" => EventType::Release, + "RepositoryEvent" => EventType::Repository, + "StatusEvent" => EventType::Status, + "TeamAddEvent" => EventType::TeamAdd, + "WatchEvent" => EventType::Watch(None), + _ => EventType::Unknown(s.into()), + }; + + Ok(ret) + } +} diff --git a/src/models/languages.rs b/src/models/languages.rs new file mode 100644 index 0000000..662fb86 --- /dev/null +++ b/src/models/languages.rs @@ -0,0 +1,51 @@ +use std::str::FromStr; +use std::convert::Infallible; + +#[derive(PartialEq, Debug)] +pub enum Language { + C, + CC, + COBOL, + + D, + + Erlang, + + FSharp, + Forth, + + Golang, + + Haskell, + + Lua, + + VisualBasic, + + Other(String), +} + +impl FromStr for Language { + type Err = Infallible; + + fn from_str(s: &str) -> Result { + let lowercase = s.clone().to_lowercase(); + + let ret = match lowercase.as_ref() { + "c" => Self::C, + "c++" | "cc" | "cpp" | "cxx" => Self::CC, + "cobol" => Self::COBOL, + "d" => Self::D, + "erlang" => Self::Erlang, + "fsharp" => Self::FSharp, + "forth" => Self::Forth, + "golang" => Self::Golang, + "haskell" => Self::Haskell, + "lua" => Self::Lua, + "visual basic" => Self::VisualBasic, + _ => Self::Other(lowercase), + }; + + Ok(ret) + } +} diff --git a/src/models/mod.rs b/src/models/mod.rs index eba3b57..a3c4c10 100644 --- a/src/models/mod.rs +++ b/src/models/mod.rs @@ -4,6 +4,7 @@ pub mod owner; pub mod reader; pub mod constraint; pub mod event_type; +pub mod languages; mod json_helpers; pub mod payloads; diff --git a/src/models/payloads/delete_payload.rs b/src/models/payloads/delete_payload.rs index ed50075..9fb46f4 100644 --- a/src/models/payloads/delete_payload.rs +++ b/src/models/payloads/delete_payload.rs @@ -1,6 +1,6 @@ use rustc_serialize::json::Json; -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq)] pub struct DeletePayload { ref_tag: String, ref_tag_type: String, diff --git a/src/models/payloads/gollum_payload.rs b/src/models/payloads/gollum_payload.rs index 7ef35a0..a6a8b10 100644 --- a/src/models/payloads/gollum_payload.rs +++ b/src/models/payloads/gollum_payload.rs @@ -1,7 +1,7 @@ use rustc_serialize::json::Json; use models::payloads::page_element::PageElement; -#[derive(Debug)] +#[derive(Debug, PartialEq)] pub struct GollumPayload { pages: Vec, } diff --git a/src/models/payloads/issue_comment_payload.rs b/src/models/payloads/issue_comment_payload.rs index 0a64baf..6a2a07c 100644 --- a/src/models/payloads/issue_comment_payload.rs +++ b/src/models/payloads/issue_comment_payload.rs @@ -1,7 +1,7 @@ use rustc_serialize::json::Json; use models::json_helpers::JsonHelper; -#[derive(Debug)] +#[derive(Debug,Eq,PartialEq)] pub struct IssueCommentPayload { comment_id: u64, issue_id: u64, diff --git a/src/models/payloads/issue_payload.rs b/src/models/payloads/issue_payload.rs index 94b7b82..a166437 100644 --- a/src/models/payloads/issue_payload.rs +++ b/src/models/payloads/issue_payload.rs @@ -1,6 +1,6 @@ use rustc_serialize::json::*; -#[derive(Debug)] +#[derive(Debug,PartialEq,Eq)] pub struct IssuePayload { action: String, issue: u64, diff --git a/src/models/payloads/page_element.rs b/src/models/payloads/page_element.rs index b340333..83397b4 100644 --- a/src/models/payloads/page_element.rs +++ b/src/models/payloads/page_element.rs @@ -2,7 +2,7 @@ use rustc_serialize::json::Json; use models::json_helpers::JsonHelper; /// This is one of the elements found within the GollumEvent payload -#[derive(Debug)] +#[derive(Debug,Eq,PartialEq)] pub struct PageElement { action: String, html_url: String, diff --git a/src/models/payloads/push_payload.rs b/src/models/payloads/push_payload.rs index 7b72289..8fc9f21 100644 --- a/src/models/payloads/push_payload.rs +++ b/src/models/payloads/push_payload.rs @@ -4,7 +4,7 @@ use rustc_serialize::json::Json; use models::payloads::ShaElement; use models::json_helpers::JsonHelper; -#[derive(Debug)] +#[derive(Debug,Eq,PartialEq)] pub struct PushPayload { head: String, /* sha hash */ refs: String, /* eg: refs/head/master */ @@ -60,4 +60,3 @@ impl PushPayload { .fold(false, |e,sum| sum || e) } } - diff --git a/src/models/payloads/sha_element.rs b/src/models/payloads/sha_element.rs index 48bb39d..d0e8101 100644 --- a/src/models/payloads/sha_element.rs +++ b/src/models/payloads/sha_element.rs @@ -1,6 +1,6 @@ use rustc_serialize::json::*; -#[derive(Debug)] +#[derive(Debug,Eq,PartialEq)] pub struct ShaElement { sha: String, email: String, @@ -62,4 +62,3 @@ impl ShaElement { &self.comment } } - diff --git a/src/models/payloads/watch_payload.rs b/src/models/payloads/watch_payload.rs index c0e4419..b41a940 100644 --- a/src/models/payloads/watch_payload.rs +++ b/src/models/payloads/watch_payload.rs @@ -1,7 +1,7 @@ use models::json_helpers::JsonHelper; use rustc_serialize::json::*; -#[derive(Debug)] +#[derive(Debug,Eq,PartialEq)] pub struct WatchPayload { action: String, } diff --git a/src/models/reader.rs b/src/models/reader.rs index 1ae53c2..c273ea5 100644 --- a/src/models/reader.rs +++ b/src/models/reader.rs @@ -1,12 +1,10 @@ -use std::io; -use std::io::{Read, Write}; +use std::io::{Read}; use std::path::PathBuf; use std::fs::File; use flate2::read::GzDecoder; /// Given a path to the json.gz file, pub fn deflate_to_contents(p: PathBuf) -> Option { - let mut stderr = io::stderr(); let pp: PathBuf = p.clone(); let ppstring: &str = pp.to_str().unwrap_or("[uncapable of unwraping]"); @@ -20,7 +18,7 @@ pub fn deflate_to_contents(p: PathBuf) -> Option { match f.read_to_end(&mut bytes) { Ok(..) => {}, Err(e) => { - writeln!(&mut stderr, "Problem loading file: {}", e).unwrap(); + println!("Problem loading file: {}", e); return None; }, } @@ -31,9 +29,8 @@ pub fn deflate_to_contents(p: PathBuf) -> Option { match d.read_to_string(&mut decomp) { Err(e) => { - writeln!(&mut stderr, - "Problem reading archive to string from error {}; archive was {}", - e, ppstring).unwrap(); + println!("Problem reading archive to string from error {}; archive was {}", + e, ppstring); return None; }, Ok(..) => {}, diff --git a/tests/fixtures/2021-01-01-1.json.gz b/tests/fixtures/2021-01-01-1.json.gz new file mode 100644 index 0000000..429a7af Binary files /dev/null and b/tests/fixtures/2021-01-01-1.json.gz differ diff --git a/tests/models/archive_test.rs b/tests/models/archive_test.rs new file mode 100644 index 0000000..3038188 --- /dev/null +++ b/tests/models/archive_test.rs @@ -0,0 +1,13 @@ +use gar::models::archive::ArchiveBuilder; + +#[test] +fn archive_test() { + let arch = ArchiveBuilder::new() + .year(1000) + .month(1) + .day(1) + .hour(1) + .finalize(); + + assert_eq!("1000-01-01-1.json.gz", arch.get_name()); +} diff --git a/tests/models/event_test.rs b/tests/models/event_test.rs new file mode 100644 index 0000000..524a5c3 --- /dev/null +++ b/tests/models/event_test.rs @@ -0,0 +1,52 @@ +use crate::DATA_SAMPLE; + +use gar::models::event::Event; + +#[test] +fn default_test() { + let event: Event = Default::default(); + + assert_eq!(0, event.get_gh_id()); + assert_eq!("", event.get_name()); +} + +#[test] +fn fixture_test() { + let events = Event::from_path(::fixture_path(DATA_SAMPLE)); + assert_eq!(1000, events.len()); +} + +#[test] +fn json_object_parsing_test() { + let json_string = r#" + { + "id":"14684219584", + "type":"CreateEvent", + "actor":{ + "id":69695756, + "login":"someone", + "display_login":"someone_login", + "gravatar_id":"", + "url":"https://api.github.com/users/someone", + "avatar_url":"https://avatars.githubusercontent.com/u/696?" + }, + "repo":{ + "id":325897057, + "name":"someone/somethingrepo", + "url":"https://api.github.com/repos/someone/totally-a-repository" + }, + "payload":{ + "ref":null, + "ref_type":"repository", + "master_branch":"master", + "description":"this is an interesting description", + "pusher_type":"user" + }, + "public":true, + "created_at":"2021-01-01T01:00:00Z" + }"#; + + let event_result = Event::into_json_object(&json_string).unwrap(); + + assert_eq!(325897057, event_result.get_gh_id()); +} diff --git a/tests/models/event_type_test.rs b/tests/models/event_type_test.rs new file mode 100644 index 0000000..b879872 --- /dev/null +++ b/tests/models/event_type_test.rs @@ -0,0 +1,11 @@ +use gar::models::event_type::EventType; + +#[test] +fn event_type_from_string_test() { + assert_eq!(EventType::Create, "CreateEvent".parse::().unwrap()); + assert_eq!(EventType::Download, "DownloadEvent".parse::().unwrap()); + + let unknown_msg = "hahahaha nope and nope".to_string(); + assert_eq!(EventType::Unknown(unknown_msg.clone()), + unknown_msg.parse::().unwrap()); +} diff --git a/tests/models/language_test.rs b/tests/models/language_test.rs new file mode 100644 index 0000000..e296e5d --- /dev/null +++ b/tests/models/language_test.rs @@ -0,0 +1,13 @@ +use gar::models::languages::Language; + +#[test] +fn match_language_test() { + assert_eq!(Language::C, "C".parse::().unwrap()); + assert_eq!(Language::CC, "C++".parse::().unwrap()); + assert_eq!(Language::CC, "cXx".parse::().unwrap()); + assert_eq!(Language::CC, "cPP".parse::().unwrap()); + assert_eq!(Language::Forth, "Forth".parse::().unwrap()); + + assert_eq!(Language::Other("blargotron".into()), + "blargotron".parse::().unwrap()); +} diff --git a/tests/tests.rs b/tests/tests.rs new file mode 100644 index 0000000..03b510c --- /dev/null +++ b/tests/tests.rs @@ -0,0 +1,23 @@ +extern crate gar; + +use std::path::PathBuf; + +// relative to tests/ +const TEST_DIR: &str = "tests"; +const DATA_DIR: &str = "fixtures"; +const DATA_SAMPLE: &str = "2021-01-01-1.json.gz"; + +fn fixture_path(s :&str) -> PathBuf { + let mut pb = PathBuf::new(); + pb.push(TEST_DIR); + pb.push(DATA_DIR); + pb.push(s); + pb +} + +mod models { + mod archive_test; + mod event_test; + mod event_type_test; + mod language_test; +}