Skip to content

Commit ddd4a25

Browse files
klaidliadonklaidliadon
authored
Merge pull request #4 from securityfirst/develop
Missing Sources
2 parents 3b85663 + 67a9b0a commit ddd4a25

File tree

17 files changed

+413
-424
lines changed

17 files changed

+413
-424
lines changed

‎country/country.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ func init() {
4646
if err != nil {
4747
log.Fatalf("Cannot open %s: %s", path, err)
4848
}
49+
defer f.Close()
4950
fipsIndex = make(map[string]string, 265)
5051
r := csv.NewReader(f)
5152
r.Comma = ' '

‎feed.go‎

Lines changed: 39 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,19 @@
11
package main
22

33
import (
4-
"encoding/json"
54
"errors"
6-
"fmt"
75
"log"
86
"time"
97

8+
"github.com/securityfirst/umbrella-api/feed"
9+
1010
"sort"
1111
"strconv"
1212
"strings"
1313

1414
"github.com/securityfirst/umbrella-api/models"
1515
"github.com/securityfirst/umbrella-api/utils"
1616

17-
"github.com/PuerkitoBio/goquery"
1817
"github.com/gin-gonic/gin"
1918
"github.com/gosexy/to"
2019
)
@@ -65,7 +64,7 @@ func (um *Umbrella) getFeedItems(sources []string, country models.Country, since
6564
var cleanSources, toRefresh, diff []int
6665
for i := range sources {
6766
inrange := to.Int64(strings.TrimSpace(sources[i]))
68-
if inrange >= 0 && inrange < SourceCount {
67+
if inrange >= 0 && inrange < feed.SourceCount {
6968
cleanSources = append(cleanSources, int(inrange))
7069
if getLastChecked[inrange] < time.Now().Add(-30*time.Minute).Unix() {
7170
toRefresh = append(toRefresh, int(inrange))
@@ -76,113 +75,47 @@ func (um *Umbrella) getFeedItems(sources []string, country models.Country, since
7675
if len(cleanSources) == 0 {
7776
err = errors.New("No Valid sources selected")
7877
} else if len(toRefresh) > 0 {
79-
for k := range toRefresh {
80-
switch src := toRefresh[k]; src {
81-
case CDC:
82-
lenItems := len(items)
83-
fmt.Println("refresh cdc")
84-
// refresh cdc
85-
if len(items) == lenItems {
86-
diff = append(diff, src)
87-
}
88-
// go um.updateLastChecked(country.Iso2, CDC, time.Now().Unix())
89-
case ReliefWeb:
90-
lenItems := len(items)
91-
body, err := makeRequest(fmt.Sprintf("https://api.reliefweb.int/v1/countries/%v", country.ReliefWeb), "get", nil)
92-
var rwResp RWResponse
93-
err = json.Unmarshal(body, &rwResp)
94-
if err != nil {
95-
utils.CheckErr(err)
96-
fmt.Println(string(body[:]))
97-
} else {
98-
go um.updateLastChecked(country.Iso2, ReliefWeb, time.Now().Unix())
99-
}
100-
if len(rwResp.Data) < 1 {
101-
utils.CheckErr(errors.New("No data received"))
102-
continue
103-
}
104-
if rwResp.Data[0].Fields.DescriptionHTML != "" {
105-
doc, err := goquery.NewDocumentFromReader(strings.NewReader(rwResp.Data[0].Fields.DescriptionHTML))
106-
if err != nil {
107-
log.Fatal(err)
108-
}
109-
s := doc.Find("h3").First()
110-
s.Next().Children().Each(func(i int, t *goquery.Selection) {
111-
href, ok := t.Contents().Attr("href")
112-
if ok {
113-
item := models.FeedItem{
114-
Title: t.Contents().Text(),
115-
URL: href,
116-
Country: country.Iso2,
117-
Source: ReliefWeb,
118-
UpdatedAt: time.Now().Unix(),
119-
}
120-
segments := strings.Split(href, "/")
121-
if len(segments) > 0 && to.Int64(segments[len(segments)-1]) != 0 {
122-
nodeUrl := fmt.Sprintf("https://api.reliefweb.int/v1/reports/%v", segments[len(segments)-1])
123-
body, err := makeRequest(nodeUrl, "get", nil)
124-
var rwReport RWReport
125-
err = json.Unmarshal(body, &rwReport)
126-
if err != nil {
127-
utils.CheckErr(err)
128-
fmt.Println(string(body[:]))
129-
} else {
130-
if rwReport.Data[0].Fields.Headline.Summary != "" {
131-
item.Description = rwReport.Data[0].Fields.Headline.Summary
132-
} else {
133-
item.Description = rwReport.Data[0].Fields.BodyHTML
134-
}
135-
item.UpdatedAt = rwReport.Data[0].Fields.Date.Changed.Unix()
136-
}
137-
138-
}
139-
items = append(items, item)
140-
go item.UpdateRelief(um.Db)
141-
}
142-
})
143-
}
144-
if len(items) == lenItems {
145-
diff = append(diff, src)
146-
}
147-
case GDASC:
148-
f := GdascFetcher{}
149-
srcItems, err := f.Fetch()
150-
if err != nil {
151-
utils.CheckErr(err)
78+
for _, src := range toRefresh {
79+
var fetcher interface {
80+
Fetch() ([]models.FeedItem, error)
81+
}
82+
var updater = (*models.FeedItem).UpdateOthers
83+
switch src {
84+
case feed.CDC:
85+
fetcher = &feed.CDCFetcher{}
86+
case feed.ReliefWeb:
87+
fetcher = &feed.ReliefWebFetcher{Country: &country}
88+
updater = (*models.FeedItem).UpdateRelief
89+
case feed.CADATA:
90+
fetcher = &feed.CadataFetcher{}
91+
case feed.GDASC:
92+
fetcher = &feed.GdascFetcher{}
93+
case feed.FCO:
94+
fetcher = &feed.FCOFetcher{}
95+
}
96+
if fetcher == nil {
97+
log.Printf("[%v] no match", src)
98+
continue
99+
}
100+
items, err := fetcher.Fetch()
101+
if err != nil {
102+
utils.CheckErr(err)
103+
continue
104+
}
105+
var change bool
106+
for i, item := range items {
107+
go updater(&items[i], um.Db)
108+
if item.Country != country.Iso2 {
152109
continue
153110
}
154-
var change bool
155-
for i, item := range srcItems {
156-
go srcItems[i].UpdateOthers(um.Db)
157-
if item.Country == country.Iso2 {
158-
items = append(items, item)
159-
if change {
160-
continue
161-
}
162-
change = true
163-
diff = append(diff, src)
164-
}
165-
}
166-
case CADATA:
167-
f := CadataFetcher{}
168-
srcItems, err := f.Fetch()
169-
if err != nil {
170-
utils.CheckErr(err)
111+
items = append(items, item)
112+
if change {
171113
continue
172114
}
173-
var change bool
174-
for i, item := range srcItems {
175-
go srcItems[i].UpdateOthers(um.Db)
176-
if item.Country == country.Iso2 {
177-
items = append(items, item)
178-
if change {
179-
continue
180-
}
181-
change = true
182-
diff = append(diff, src)
183-
}
184-
}
115+
change = true
116+
diff = append(diff, src)
185117
}
118+
go um.updateLastChecked(country.Iso2, src, time.Now().Unix())
186119
}
187120
}
188121
if len(diff) > 0 {
@@ -222,13 +155,3 @@ func (slice SortFeedByDate) Less(i, j int) bool {
222155
func (slice SortFeedByDate) Swap(i, j int) {
223156
slice[i], slice[j] = slice[j], slice[i]
224157
}
225-
226-
const (
227-
ReliefWeb = iota
228-
FCO
229-
UN
230-
CDC
231-
GDASC
232-
CADATA
233-
SourceCount
234-
)

‎feed_cadata.go‎ renamed to ‎feed/cadata.go‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package main
1+
package feed
22

33
import (
44
"encoding/xml"
@@ -14,16 +14,14 @@ import (
1414
type CadataFetcher struct{}
1515

1616
func (g *CadataFetcher) Fetch() ([]models.FeedItem, error) {
17+
const baseURL = "https://cadatacatalog.state.gov/storage/f/2013-11-24T21%3A00%3A"
1718
var feeds []models.FeedItem
18-
for _, src := range []string{
19-
"https://cadatacatalog.state.gov/storage/f/2013-11-24T21%3A00%3A30.424Z/tas.xml",
20-
"https://cadatacatalog.state.gov/storage/f/2013-11-24T21%3A00%3A58.223Z/tws.xml",
21-
} {
22-
body, err := makeRequest(src, http.MethodGet, nil)
19+
for _, src := range []string{"30.424Z/tas.xml", "58.223Z/tws.xml"} {
20+
body, err := makeRequest(baseURL+src, http.MethodGet, nil)
2321
if err != nil {
2422
return nil, err
2523
}
26-
var v CadataResp
24+
var v cadataResp
2725
if err := xml.Unmarshal(body, &v); err != nil {
2826
return nil, err
2927
}
@@ -51,7 +49,7 @@ func (g *CadataFetcher) Fetch() ([]models.FeedItem, error) {
5149
return feeds, nil
5250
}
5351

54-
type CadataResp struct {
52+
type cadataResp struct {
5553
Link []string `xml:"channel>item>link"`
5654
Description []string `xml:"channel>item>description"`
5755
Title []string `xml:"channel>item>title"`

‎feed/cdc.go‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
package feed
2+
3+
import (
4+
"encoding/xml"
5+
"log"
6+
"net/http"
7+
"regexp"
8+
"strings"
9+
"time"
10+
11+
"github.com/securityfirst/umbrella-api/country"
12+
"github.com/securityfirst/umbrella-api/models"
13+
)
14+
15+
type CDCFetcher struct{}
16+
17+
var cdcCountry = regexp.MustCompile(`.* in (.*)$`)
18+
19+
func (c *CDCFetcher) Fetch() ([]models.FeedItem, error) {
20+
body, err := makeRequest("https://tools.cdc.gov/api/v2/resources/media/285740.rss", http.MethodGet, nil)
21+
if err != nil {
22+
return nil, err
23+
}
24+
var resp cdcResp
25+
if err := xml.Unmarshal(body, &resp); err != nil {
26+
return nil, err
27+
}
28+
var feeds = make([]models.FeedItem, 0, len(resp.Item))
29+
for _, v := range resp.Item {
30+
match := cdcCountry.FindStringSubmatch(v.Title)
31+
if len(match) != 2 {
32+
continue
33+
}
34+
name := strings.TrimSpace(match[1])
35+
t, _ := time.Parse(time.RFC1123, v.PubDate)
36+
c, err := country.ByName(name)
37+
if err != nil {
38+
log.Printf("CDCFeed - Country %q: %s", name, err)
39+
continue
40+
}
41+
feeds = append(feeds, models.FeedItem{
42+
Title: v.Title,
43+
Description: v.Description,
44+
URL: v.Link,
45+
Country: strings.ToLower(c.Codes.Alpha2),
46+
Source: CDC,
47+
UpdatedAt: t.Unix(),
48+
})
49+
}
50+
return feeds, nil
51+
}
52+
53+
type cdcResp struct {
54+
Item []struct {
55+
Text string `xml:",chardata"`
56+
Title string `xml:"title"`
57+
Description string `xml:"description"`
58+
Link string `xml:"link"`
59+
GUID struct {
60+
Text string `xml:",chardata"`
61+
IsPermaLink string `xml:"isPermaLink,attr"`
62+
} `xml:"guid"`
63+
PubDate string `xml:"pubDate"`
64+
Category string `xml:"category"`
65+
} `xml:"channel>item"`
66+
}

‎feed/common.go‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
package feed
2+
3+
import (
4+
"io"
5+
"io/ioutil"
6+
"net/http"
7+
"strings"
8+
"time"
9+
10+
"github.com/securityfirst/umbrella-api/models"
11+
)
12+
13+
type Fetcher interface {
14+
Fetch() ([]models.FeedItem, error)
15+
}
16+
17+
var client = http.Client{
18+
Timeout: time.Second * 10,
19+
}
20+
21+
func makeRequest(uri string, method string, requestBody io.Reader) ([]byte, error) {
22+
req, err := http.NewRequest(strings.ToUpper("GET"), uri, requestBody)
23+
if err != nil {
24+
return nil, err
25+
}
26+
req.Close = true
27+
resp, err := client.Do(req)
28+
if err != nil {
29+
return nil, err
30+
}
31+
defer resp.Body.Close()
32+
return ioutil.ReadAll(resp.Body)
33+
}
34+
35+
// List of Sources
36+
const (
37+
ReliefWeb = iota
38+
FCO
39+
UN
40+
CDC
41+
GDASC
42+
CADATA
43+
SourceCount
44+
)

‎feed_all_test.go‎ renamed to ‎feed/common_test.go‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,11 @@
1-
package main
1+
package feed
22

33
import (
44
"testing"
5+
56
"github.com/securityfirst/umbrella-api/models"
67
)
78

8-
type Fetcher interface {
9-
Fetch() ([]models.FeedItem, error)
10-
}
11-
129
func baseTest(t *testing.T, f Fetcher) {
1310
items, err := f.Fetch()
1411
if err != nil {
@@ -18,7 +15,7 @@ func baseTest(t *testing.T, f Fetcher) {
1815
}
1916

2017
func TestRefiweb(t *testing.T) {
21-
baseTest(t, &RefiWebFetcher{&models.Country{ReliefWeb: 241, Iso2: "UA"}})
18+
baseTest(t, &ReliefWebFetcher{&models.Country{ReliefWeb: 241, Iso2: "UA"}})
2219
}
2320

2421
func TestGdasc(t *testing.T) {
@@ -28,3 +25,11 @@ func TestGdasc(t *testing.T) {
2825
func TestCadata(t *testing.T) {
2926
baseTest(t, &CadataFetcher{})
3027
}
28+
29+
func TestCDC(t *testing.T) {
30+
baseTest(t, &CDCFetcher{})
31+
}
32+
33+
func TestFCO(t *testing.T) {
34+
baseTest(t, &FCOFetcher{})
35+
}

0 commit comments

Comments
 (0)