From fdd314f0885f47275ad78e620fa0fa1674899cc9 Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 11:57:35 +0200 Subject: [PATCH 1/7] Adds function to load data from the provided json file into the DB. --- db.sqlite3 | Bin 151552 -> 151552 bytes .../migrations/0002_auto_20181227_1125.py | 35 ++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 departures/migrations/0002_auto_20181227_1125.py diff --git a/db.sqlite3 b/db.sqlite3 index d3ec30e84c66ce80310e81d3d0d50f260e736194..7923437fadbce38faf6770d0f74f08d9a5ce0cad 100644 GIT binary patch delta 3998 zcmeHJ`)}LT760Dz^)rrrotGUy;z!c7bzL|9+DYs*q-kDFlQ;>5cC0|_x{2GSYm<;i zp=C4)xk<*G64ArPfNCJNAu%42C`@!?PYs56Xh@s(!zdDhYGOm6ZD!PK;y|aV@PS5~-}FCF92afX)z{V~8J zzgNc6Z=WhPun4mp8_HjwT?H*zY+%*QqzlLMy7z#~z+%j(iv~u9CnO>h$4B!O+`xj& zpo=HR^J9LefrXi>pAU|WOpf}>3@pk(7hKB4TWqkdVkIc)+ zBSw`mOORGmfL3D#tp-1>s*hIi8dNLuEKR(5$C)T(lp^JT!go0yDSWBWW@80S)@Ljq zTVAlVnr||`ZhX-2E#-jvzIv^40JUk>XdOtI-LV*(KaTAS609|Nc!6CRSZR-mp`-cA z&|q#NPv1g){X0W_yD}O4;R0*Ms&BA-Ak^Nmt>cO-T3Rz5p{~BJon0-N%|dr zvCaQW=Wh#agDqk|&7AJ$6q{+p=jPcaquxgN?mXMF{B?1}Sk@qwo^z*CY~D7J+kYhg zg{}96LL6FPK|FN_tBTJ!8_e$h7@N_Ch6nREd`_LppB);^4#?Ke@6Xla&UqGheoo34 zn73s`S{a#HpEkR*(&&yaHoDSkS?V?2cK5$< z+SxPH>~V&d-Nk;6Puy8ZBpUy_tPS zH7E*mV0@lU7MoHgeg4c81k5_zQLLA6n{>GScwt`J$jmbf-agMl7(M{!M z_mwSdW*0tm3k#if+`49QhRASFd&UulLIPX!5i4Usbv^5!QH&iglf)&xwlzcU8?QB_VwV zpUu6@9#+w?6@CeGW_)=oBw@~ktOe>}&WOV$;KBPlz=8W(zyfmyJlp~)%baRbBH^$P zSqYTlxfXE29N^uXp+Z+a-AcL)XHrt|ajSg&f(uM)d``hFt)PLJ!s}Y07Q{*1t3ldY zD{e9=(Qvp5gDEJZ7ha3_x=E=Chdo$S0d9Pr6_O z%HA?kw~eA6!Vk8=W)O!k)=7E?@jxfkfjEEKPdLO>DopR0MKem&Vy{L7`%85NV)J0aV#Rs}b{~CO)i|!>Q zx(WE**whX67V%XbchBiAs9J`%R$PT|r6Gie+n@}@SMU$rgw!td^iVcCv8RXZT#5fI z1}|0Kv7ERjuE1w|$mI@vuSYhKxE#Y7h=J(W5meKn4}a{1Aih>X-BcpMN{U{5HbdcL z@Xr}~(}PvLRFH1m(@VCx@Ki6k-Ht!g&z<;DFT^Zj+Y0dZ?u)>uMF;Ndqfp!N{yy^C zhA-)REB?KYY`5U4CfKN^#8&*E2}*FZAF4q#Oo@tG{B|*Hw1}imG!N#yl4yD}Bt?S2 zm#Ar|l|UniIMK8QL=8SvLf$t~6mf}o?6Zh#Vk2%Xg@i>!bp$=wR1f9Y>|JJ;8N>$s zekq|C!M95(#`RcHMwM8HJIct{TD-H2Dk3?c?`!bGGHSRg+~6X4C0^?yy)b^$MVW{2 zMHjsf63+5IfHiK4y@DFhPo6z$o$%uWZfZxLPBI@lTcP@NIZ@V&ulwW((lbvB57jt` zyUW1~qMVv1DF=d^V)WzXeu-{UxbV$#s4Wy_CM7J%*^?hWj&EtQS#^TOMJaZA2&ED{ z;i0}L#wR?KX%SxVP-IR*HB6C79~_^kl~45vxI9A~c!!q~u@klus6u?dL26XA$V8ID zhH)PmDWFN}!6_dZu~I}yH6|>Sj0DAu%nvmdVbY7>TP}hO-vz>mhx`v$ z^HaEhwhB3U1v4p))GUv;LUft3F`?ke3TbzC3V$GVVD==wTOo@wdlLo&^6l)`sc1gr zeR7s0XKzFikTK4lAlNiDJ$sx|Nvnz38!!@-&S$U3y+PSfv&ZnRAT+`3B)$@aNH{QO zmXxR#O?J7dJg@K<`J&tc^5~Zz-`VmI^l~*iSAd6_@EwEm_N}P^P?ulnm={}8?y=7g@r_o zt#J1xshyf=WX$Pkb&~36r&Fg!I--uT89R2WV zHP(=|n7REpyRUfr#~s0Zxl54$D}TW}Uy(nSKV*i6<+Ji1<+tQFBzTVxNV=6`W zZpb!Bb=r4a`7k_$Uq1}5;hBeFsWp3@@GqsJTEnFX#}2`n-i^jeeC7}Ys+pZnRUq6m z0c&nfwzW>7rv6vj+hee2sXu%DmJlvG3|pBYTmO>>K?nGwAH&;V>VwEPB`>$@Y+}A8 ziuyC)XT9LI^he)=XT?5UL~rXie+d8dKHg)7g=n#nSNh*_l4q?LD3|Sb${;Z*=}g*uxusA=;Lcm*>)4Pil|*h^||dV}ksS+{<3&vcdW%>l4=P z*2?V9vQK9p&EAwdcwn6{cqjaQ7X7<-Jj7;_Ed zhUX3Y4L2L|^ncO6tba&c(}y)*aBT(pkkZ@mX=V zSSQ-E$T0Z@*+YT^U=WVO7AO-w6JE!YPU0fz9Ncyp7T~*skZY_>+a$s3aus6o64;At zJ^tESpTFMYtM$~@u4;?2xoiogWl0EenHNW)5|%6C6%`7v_%riT`osc*REuHJc`H0z=3BbKoWgvGu}7|zWKqlN!4~-qg|xR z*@9`KBviRvWowhEWJ^mDw+^wudvb||qz!oNG8Evl0%BD|%C1T4C82}~uZ*_ExwD1A znzSSdZWhAIRP;c+&DnH+dn(r1Iho<~T-QcZ@kGp@&O*~56cx<`LOZNW>r|}D@Xax> zm(CPf(YimD=-QV`?&}PtMM?0mK=A!Mk~g=UFHKZA2cF6&dEGPZX)bHiAPE&bCoO>u z(VZ;L+2Wt(KbRI~pmx2qud3-fy|f#T<`N&F58;*qQZRcr1nG}7>)3pO@ycVPV8stV z01Kf%#4BS^ls_|%Ob^x32l3bl6t&Jgz3RR*NDoLtfOXllR=8f|8N-eA0ZC|J2yIBR zW#U}1GZk+^I-ivApv+SQ(hI_yCc~ek2)slA-Z3y z@$IqHK4(*+CAqhq4NI(b4!w<&&Cx*jY4bAW^4pasb^WwUby~P79_va(_wqR_(&o`e zJMrmDP>8#SSUuQ?8VF6{>R~9F`l1*8v_tLgg)@tgX*JreF~Owq0UIe_R!x)I5N9UP zxiJ;nGh2?D5f4qM;)Rm~tIj-U44a4Uo%^UwJDR~1T&poe_h{VnUE#`9Hb{5l3!~t) z&z7o}GJas>arex_saD-331QZV8&YxSnrLEA^xNv2Q4G*~)ov=p_a`9VJ*)8m-Koh; z>vHs;hf*;>WB3)DI=u0`Hc{r>Fm08DMn+%LdOM2UqfleNjq;OkADR%RcKXv{U)73=rMQrJEMCVa`u_?VM?Lx47Grmy07Q@pb! z$wVKRyMyZJc5KceE_FTK7NFZy0xwg??7I%<61r8zZ8|5YxMfHoK)2w3zDEk?rse?M zoGIluGrlcn z!y;21Ki#Ab#^nB+vzpV~hv;1zGi8R!nel_;c|_04;}EOqMy#2DYSu*?wEc)MkIcVj z9JEr0>78oUg)_Qp(%-B!In`QsX!XZC*y6v2fHmVmdi(#esX9Y+y_RAI$?LXNCe%E> znd|8~HKejz*gd5^dT+85JM6?Aa?`cy)xn{*+4U)|hOU{0*{u9hrF7vgFR3O@Zo Date: Thu, 27 Dec 2018 12:37:06 +0200 Subject: [PATCH 2/7] Adds script to write filtered API data as CSV. Updates requirements.txt to comply with used libraries. --- adventurous_departures.csv | 25 ++++++++++++++++ data_collection_script.py | 61 ++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 ++ 3 files changed, 88 insertions(+) create mode 100644 adventurous_departures.csv create mode 100644 data_collection_script.py diff --git a/adventurous_departures.csv b/adventurous_departures.csv new file mode 100644 index 0000000..7c36d73 --- /dev/null +++ b/adventurous_departures.csv @@ -0,0 +1,25 @@ +Name,Start Date,Finish Date,Category +New Zealand Encompassed,2018-08-31,2018-09-10,Adventurous +Cambodia Overland,2018-08-31,2018-09-10,Adventurous +Peru Overland,2018-08-31,2018-09-10,Adventurous +Vietnam Safari,2018-08-31,2018-09-10,Adventurous +Brazil Trek,2018-08-31,2018-09-10,Adventurous +Cambodia Multisport,2018-08-31,2018-09-10,Adventurous +Morocco Safari,2018-08-31,2018-09-10,Adventurous +Galapagos Encompassed,2018-08-31,2018-09-10,Adventurous +Galapagos Discovery,2018-08-31,2018-09-10,Adventurous +New Zealand Adventure,2018-08-31,2018-09-10,Adventurous +Galapagos Trek,2018-08-31,2018-09-10,Adventurous +Brazil Encompassed,2018-08-31,2018-09-10,Adventurous +Kenya Express,2018-08-31,2018-09-10,Adventurous +Galapagos Discovery,2018-08-31,2018-09-10,Adventurous +Sri Lanka Adventure,2018-08-31,2018-09-10,Adventurous +Brazil Adventure,2018-08-31,2018-09-10,Adventurous +Brazil Overland,2018-08-31,2018-09-10,Adventurous +Kenya Encompassed,2018-08-31,2018-09-10,Adventurous +Morocco Discovery,2018-08-31,2018-09-10,Adventurous +Vietnam Trek,2018-08-31,2018-09-10,Adventurous +New Zealand Safari,2018-08-31,2018-09-10,Adventurous +Vietnam Encompassed,2018-08-31,2018-09-10,Adventurous +Ethopia Express,2018-08-31,2018-09-10,Adventurous +Sri Lanka Safari,2018-08-31,2018-09-10,Adventurous diff --git a/data_collection_script.py b/data_collection_script.py new file mode 100644 index 0000000..346a5b7 --- /dev/null +++ b/data_collection_script.py @@ -0,0 +1,61 @@ +import pandas as pd +import re +import requests +import json +import datetime + +API_URL = 'http://127.0.0.1:8000/departures/' + +CSV_FILENAME = "adventurous_departures.csv" + + +def get_api_content(url): + response = requests.get(url) + return response.content + + +def get_departures(): + results = [] + json_data = get_api_content(API_URL) + + apidata = json.loads(json_data) + results.extend(apidata['results']) + + next_data = apidata.get('next') + + while next_data: + apidata = json.loads(get_api_content(apidata['next'])) + next_data = apidata.get('next') + results.extend(apidata['results']) + + return results + + +def filter_departures(in_data): + df = pd.DataFrame.from_dict(in_data) + df['start_date'] = df['start_date'].astype('datetime64[ns]') + df = df[['name', 'start_date', 'finish_date', 'category']] + filtered_df = df[(df.start_date > datetime.datetime(2018, 6, 1)) & + (df.category == "Adventurous")] + + return filtered_df + + +def write_csv(data_frame): + data_frame.rename(columns={ + 'name': 'Name', + 'start_date': 'Start Date', + 'finish_date': 'Finish Date', + 'category': 'Category'}, + inplace=True) + + data_frame.to_csv(CSV_FILENAME, index=False) + + +def main(): + apidata = get_departures() + filtered_data = filter_departures(apidata) + write_csv(filtered_data) + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt index 506c7d5..9f15821 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ Django==2.1.4 djangorestframework==3.7.7 +requests +pandas From 699c14c22a196e2993eed00428387be2b793230f Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 12:50:00 +0200 Subject: [PATCH 3/7] Updates the script to handle naming of CSV headers in flexible way. --- adventurous_departures.csv | 50 +++++++++++++++++++------------------- data_collection_script.py | 14 +++++------ 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/adventurous_departures.csv b/adventurous_departures.csv index 7c36d73..3083df7 100644 --- a/adventurous_departures.csv +++ b/adventurous_departures.csv @@ -1,25 +1,25 @@ -Name,Start Date,Finish Date,Category -New Zealand Encompassed,2018-08-31,2018-09-10,Adventurous -Cambodia Overland,2018-08-31,2018-09-10,Adventurous -Peru Overland,2018-08-31,2018-09-10,Adventurous -Vietnam Safari,2018-08-31,2018-09-10,Adventurous -Brazil Trek,2018-08-31,2018-09-10,Adventurous -Cambodia Multisport,2018-08-31,2018-09-10,Adventurous -Morocco Safari,2018-08-31,2018-09-10,Adventurous -Galapagos Encompassed,2018-08-31,2018-09-10,Adventurous -Galapagos Discovery,2018-08-31,2018-09-10,Adventurous -New Zealand Adventure,2018-08-31,2018-09-10,Adventurous -Galapagos Trek,2018-08-31,2018-09-10,Adventurous -Brazil Encompassed,2018-08-31,2018-09-10,Adventurous -Kenya Express,2018-08-31,2018-09-10,Adventurous -Galapagos Discovery,2018-08-31,2018-09-10,Adventurous -Sri Lanka Adventure,2018-08-31,2018-09-10,Adventurous -Brazil Adventure,2018-08-31,2018-09-10,Adventurous -Brazil Overland,2018-08-31,2018-09-10,Adventurous -Kenya Encompassed,2018-08-31,2018-09-10,Adventurous -Morocco Discovery,2018-08-31,2018-09-10,Adventurous -Vietnam Trek,2018-08-31,2018-09-10,Adventurous -New Zealand Safari,2018-08-31,2018-09-10,Adventurous -Vietnam Encompassed,2018-08-31,2018-09-10,Adventurous -Ethopia Express,2018-08-31,2018-09-10,Adventurous -Sri Lanka Safari,2018-08-31,2018-09-10,Adventurous +Category,Finish Date,Name,Start Date +Adventurous,2018-09-10,New Zealand Encompassed,2018-08-31 +Adventurous,2018-09-10,Cambodia Overland,2018-08-31 +Adventurous,2018-09-10,Peru Overland,2018-08-31 +Adventurous,2018-09-10,Vietnam Safari,2018-08-31 +Adventurous,2018-09-10,Brazil Trek,2018-08-31 +Adventurous,2018-09-10,Cambodia Multisport,2018-08-31 +Adventurous,2018-09-10,Morocco Safari,2018-08-31 +Adventurous,2018-09-10,Galapagos Encompassed,2018-08-31 +Adventurous,2018-09-10,Galapagos Discovery,2018-08-31 +Adventurous,2018-09-10,New Zealand Adventure,2018-08-31 +Adventurous,2018-09-10,Galapagos Trek,2018-08-31 +Adventurous,2018-09-10,Brazil Encompassed,2018-08-31 +Adventurous,2018-09-10,Kenya Express,2018-08-31 +Adventurous,2018-09-10,Galapagos Discovery,2018-08-31 +Adventurous,2018-09-10,Sri Lanka Adventure,2018-08-31 +Adventurous,2018-09-10,Brazil Adventure,2018-08-31 +Adventurous,2018-09-10,Brazil Overland,2018-08-31 +Adventurous,2018-09-10,Kenya Encompassed,2018-08-31 +Adventurous,2018-09-10,Morocco Discovery,2018-08-31 +Adventurous,2018-09-10,Vietnam Trek,2018-08-31 +Adventurous,2018-09-10,New Zealand Safari,2018-08-31 +Adventurous,2018-09-10,Vietnam Encompassed,2018-08-31 +Adventurous,2018-09-10,Ethopia Express,2018-08-31 +Adventurous,2018-09-10,Sri Lanka Safari,2018-08-31 diff --git a/data_collection_script.py b/data_collection_script.py index 346a5b7..1f3f7a4 100644 --- a/data_collection_script.py +++ b/data_collection_script.py @@ -31,10 +31,14 @@ def get_departures(): return results +def convert_to_title_case(input_string): + temp = re.sub(r'_|-', " ", input_string) + return temp.title() + + def filter_departures(in_data): df = pd.DataFrame.from_dict(in_data) df['start_date'] = df['start_date'].astype('datetime64[ns]') - df = df[['name', 'start_date', 'finish_date', 'category']] filtered_df = df[(df.start_date > datetime.datetime(2018, 6, 1)) & (df.category == "Adventurous")] @@ -42,13 +46,7 @@ def filter_departures(in_data): def write_csv(data_frame): - data_frame.rename(columns={ - 'name': 'Name', - 'start_date': 'Start Date', - 'finish_date': 'Finish Date', - 'category': 'Category'}, - inplace=True) - + data_frame.rename(columns=convert_to_title_case, inplace=True) data_frame.to_csv(CSV_FILENAME, index=False) From 08024cc2a93cb2b5f1a213d0e5cf344227b7f1ed Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 13:43:53 +0200 Subject: [PATCH 4/7] Adds tests for the data collection script. --- tests_data_collection_script.py | 64 +++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 tests_data_collection_script.py diff --git a/tests_data_collection_script.py b/tests_data_collection_script.py new file mode 100644 index 0000000..3e292a7 --- /dev/null +++ b/tests_data_collection_script.py @@ -0,0 +1,64 @@ +import unittest +import data_collection_script as dc + + +class TestCollectionScript(unittest.TestCase): + + def test_title_case_single_word(self): + assert(dc.convert_to_title_case("single") == + "Single") + + def test_title_case_two_words(self): + assert(dc.convert_to_title_case("double_word") == + "Double Word") + + def test_title_case_multiple_words(self): + assert(dc.convert_to_title_case( + "multiple_word-with-different_punctuation") == + "Multiple Word With Different Punctuation") + + def test_filter_valid_date_and_category(self): + input_data = [{ + "name": "New Zealand Safari", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Marine" + }, { + "name": "New Zealand Encompassed", + "start_date": "2018-06-02", + "finish_date": "2018-06-10", + "category": "Adventurous" + }, { + "name": "Australia Encompassed", + "start_date": "2018-06-02", + "finish_date": "2018-06-10", + "category": "Marine" + }] + + out_df = dc.filter_departures(input_data) + assert(len(out_df) == 1) + + def test_filter_invalid_category(self): + input_data = [{ + "name": "New Zealand Safari", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Marine" + }, { + "name": "New Zealand Encompassed", + "start_date": "2018-06-02", + "finish_date": "2018-06-10", + "category": "Random" + }, { + "name": "Australia Encompassed", + "start_date": "2018-06-02", + "finish_date": "2018-06-10", + "category": "Marine" + }] + + out_df = dc.filter_departures(input_data) + assert(len(out_df) == 0) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From b13b757e4d5631f590daebbed44b00c4e562da97 Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 14:02:14 +0200 Subject: [PATCH 5/7] Adds documentation for the script's functions. Adds additional test checks. --- data_collection_script.py | 27 +++++++++++++++++++++++++-- tests_data_collection_script.py | 6 ++++++ 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/data_collection_script.py b/data_collection_script.py index 1f3f7a4..41fe301 100644 --- a/data_collection_script.py +++ b/data_collection_script.py @@ -10,11 +10,18 @@ def get_api_content(url): + """ + Returns the content of the given URL. + """ response = requests.get(url) return response.content def get_departures(): + """ + Returns the 'departures' data from the URL, based + on the provided information of parsing the JSON data. + """ results = [] json_data = get_api_content(API_URL) @@ -32,11 +39,18 @@ def get_departures(): def convert_to_title_case(input_string): + """ + Returns the given string in title case formatting. + """ temp = re.sub(r'_|-', " ", input_string) return temp.title() def filter_departures(in_data): + """ + Filters the given data with the desired filters using Pandas. + Returns a dataframe for writing to CSV + """ df = pd.DataFrame.from_dict(in_data) df['start_date'] = df['start_date'].astype('datetime64[ns]') filtered_df = df[(df.start_date > datetime.datetime(2018, 6, 1)) & @@ -45,12 +59,21 @@ def filter_departures(in_data): return filtered_df -def write_csv(data_frame): +def write_csv(data_frame, csv_name=CSV_FILENAME): + """ + Writes the given data frame into a CSV file with the given name. + """ data_frame.rename(columns=convert_to_title_case, inplace=True) - data_frame.to_csv(CSV_FILENAME, index=False) + data_frame.to_csv(csv_name, index=False) def main(): + """ + The main function of the script. + Calls all required functions to complete the task + of getting API data, filtering it and writing it to + a CSV file. + """ apidata = get_departures() filtered_data = filter_departures(apidata) write_csv(filtered_data) diff --git a/tests_data_collection_script.py b/tests_data_collection_script.py index 3e292a7..cccd73b 100644 --- a/tests_data_collection_script.py +++ b/tests_data_collection_script.py @@ -1,5 +1,6 @@ import unittest import data_collection_script as dc +from datetime import datetime class TestCollectionScript(unittest.TestCase): @@ -37,6 +38,11 @@ def test_filter_valid_date_and_category(self): out_df = dc.filter_departures(input_data) assert(len(out_df) == 1) + assert(out_df.iloc[0, 0] == input_data[1]['category']) + assert(out_df.iloc[0, 1] == input_data[1]['finish_date']) + assert(out_df.iloc[0, 2] == input_data[1]['name']) + assert(out_df.iloc[0, 3] == + datetime.strptime(input_data[1]['start_date'], '%Y-%m-%d')) def test_filter_invalid_category(self): input_data = [{ From 22113d658561ec70463f663c6a44dd706081e1f5 Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 14:51:15 +0200 Subject: [PATCH 6/7] Adds additional test for GET api data. --- tests_data_collection_script.py | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests_data_collection_script.py b/tests_data_collection_script.py index cccd73b..827b1c6 100644 --- a/tests_data_collection_script.py +++ b/tests_data_collection_script.py @@ -1,6 +1,9 @@ import unittest import data_collection_script as dc from datetime import datetime +from unittest.mock import Mock +from unittest.mock import patch +import json class TestCollectionScript(unittest.TestCase): @@ -65,6 +68,32 @@ def test_filter_invalid_category(self): out_df = dc.filter_departures(input_data) assert(len(out_df) == 0) + @patch('data_collection_script.requests.get') + def tests_get_departures_single_get(self, mock_get): + + mock_data = { + "count": 150, + "next": None, + "previous": None, + "results": [ + { + "name": "New Zealand Safari", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Marine" + }, + { + "name": "New Zealand Encompassed", + "start_date": "2018-08-31", + "finish_date": "2018-09-10", + "category": "Adventurous" + }] + } + + mock_get.return_value.content = json.dumps(mock_data) + + response = dc.get_departures() + assert(response == mock_data['results']) if __name__ == '__main__': unittest.main() \ No newline at end of file From e62e8263cd082605f5e0deecc22c9ea90a906dee Mon Sep 17 00:00:00 2001 From: Vlado Paskov Date: Thu, 27 Dec 2018 16:31:26 +0200 Subject: [PATCH 7/7] Adds additional tests for getting API data and adding library to requirements. --- requirements.txt | 1 + tests_data_collection_script.py | 69 ++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 6 deletions(-) diff --git a/requirements.txt b/requirements.txt index 9f15821..9749344 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ Django==2.1.4 djangorestframework==3.7.7 requests pandas +requests_testing diff --git a/tests_data_collection_script.py b/tests_data_collection_script.py index 827b1c6..6156dec 100644 --- a/tests_data_collection_script.py +++ b/tests_data_collection_script.py @@ -4,6 +4,7 @@ from unittest.mock import Mock from unittest.mock import patch import json +import requests_testing class TestCollectionScript(unittest.TestCase): @@ -68,10 +69,9 @@ def test_filter_invalid_category(self): out_df = dc.filter_departures(input_data) assert(len(out_df) == 0) - @patch('data_collection_script.requests.get') - def tests_get_departures_single_get(self, mock_get): - - mock_data = { + @requests_testing.activate + def tests_get_departures_single_get(self): + mock_data = { "count": 150, "next": None, "previous": None, @@ -88,12 +88,69 @@ def tests_get_departures_single_get(self, mock_get): "finish_date": "2018-09-10", "category": "Adventurous" }] - } + } - mock_get.return_value.content = json.dumps(mock_data) + requests_testing.add( + request={'url': 'http://127.0.0.1:8000/departures/'}, + response={'body': json.dumps(mock_data)} + ) response = dc.get_departures() assert(response == mock_data['results']) + @requests_testing.activate + def tests_get_departures_get_with_next_data(self): + mock_data = { + "count": 150, + "next": "http://127.0.0.1:8000/departures/?limit=50&offset=50", + "previous": None, + "results": [ + { + "name": "New Zealand Safari", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Marine" + }, + { + "name": "New Zealand Encompassed", + "start_date": "2018-08-31", + "finish_date": "2018-09-10", + "category": "Adventurous" + }] + } + + mock_data_next = { + "count": 150, + "next": None, + "previous": None, + "results": [{ + "name": "Brazil Adventure", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Classic" + }, + { + "name": "Vietnam Encompassed", + "start_date": "2018-04-03", + "finish_date": "2018-04-13", + "category": "Marine" + }] + } + + requests_testing.add( + request={'url': 'http://127.0.0.1:8000/departures/'}, + response={'body': json.dumps(mock_data)} + ) + requests_testing.add( + request={'url': + 'http://127.0.0.1:8000/departures/?limit=50&offset=50'}, + response={'body': json.dumps(mock_data_next)} + ) + + # mock_get.return_value.content = json.dumps(mock_data) + + response = dc.get_departures() + assert(response == mock_data['results'] + mock_data_next['results']) + if __name__ == '__main__': unittest.main() \ No newline at end of file