Refactoring
parent
b58a82da5e
commit
eed6a7d00d
29
scraper.go
29
scraper.go
|
@ -5,7 +5,6 @@ import (
|
|||
"fmt"
|
||||
"gopkg.in/gcfg.v1"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
@ -21,29 +20,9 @@ func main() {
|
|||
// Get Twitter Client
|
||||
twitter = WorstCaptcha.NewTwitter(config)
|
||||
|
||||
// Do twitter search and loop
|
||||
for _, tweet := range twitter.DoSearch() {
|
||||
fmt.Printf("\n=================\n")
|
||||
|
||||
// Type assert entities to be a map
|
||||
entities := tweet["entities"].(map[string]interface{})
|
||||
|
||||
// Type assert media to be a list
|
||||
mediaList := entities["media"].([]interface{})
|
||||
|
||||
// Type assert the media to be a map
|
||||
media := mediaList[0].(map[string]interface{})
|
||||
|
||||
// Get the link
|
||||
imageUrl := media["media_url_https"]
|
||||
|
||||
// Parts of word
|
||||
parts := strings.Split(tweet.Text(), " ")
|
||||
|
||||
// Build word without link
|
||||
word := strings.Join(parts[0:len(parts)-1], " ")
|
||||
|
||||
fmt.Println(imageUrl)
|
||||
fmt.Println(word)
|
||||
// Get images
|
||||
for wordlink, _ := range twitter.GetImages() {
|
||||
fmt.Println("===================")
|
||||
fmt.Println(wordlink)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -7,6 +7,7 @@ import (
|
|||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func NewTwitter(config Config) Twitter {
|
||||
|
@ -27,7 +28,10 @@ type Twitter struct {
|
|||
client *twittergo.Client
|
||||
}
|
||||
|
||||
func (t *Twitter) DoSearch() []twittergo.Tweet {
|
||||
//
|
||||
// Searching
|
||||
//
|
||||
func (t *Twitter) doSearch() []twittergo.Tweet {
|
||||
var (
|
||||
err error
|
||||
req *http.Request
|
||||
|
@ -71,3 +75,41 @@ func (t *Twitter) DoSearch() []twittergo.Tweet {
|
|||
|
||||
return results.Statuses()
|
||||
}
|
||||
|
||||
//
|
||||
// Get Images from twitter
|
||||
//
|
||||
func (t *Twitter) GetImages() WordLinks {
|
||||
var wls = WordLinks{}
|
||||
|
||||
for _, tweet := range t.doSearch() {
|
||||
wls[t.filterTweet(tweet)] = true
|
||||
}
|
||||
|
||||
return wls
|
||||
}
|
||||
|
||||
//
|
||||
// Convert a single tweet to a set of word and link
|
||||
//
|
||||
func (t *Twitter) filterTweet(tweet twittergo.Tweet) WordLink {
|
||||
// Type assert entities to be a map
|
||||
entities := tweet["entities"].(map[string]interface{})
|
||||
|
||||
// Type assert media to be a list
|
||||
mediaList := entities["media"].([]interface{})
|
||||
|
||||
// Type assert the media to be a map
|
||||
media := mediaList[0].(map[string]interface{})
|
||||
|
||||
// Get the link
|
||||
imageUrl := media["media_url_https"].(string)
|
||||
|
||||
// Parts of word
|
||||
parts := strings.Split(tweet.Text(), " ")
|
||||
|
||||
// Build word without link
|
||||
word := strings.Join(parts[0:len(parts)-1], " ")
|
||||
|
||||
return WordLink{word, imageUrl}
|
||||
}
|
||||
|
|
|
@ -5,4 +5,4 @@ type WordLink struct {
|
|||
Link string
|
||||
}
|
||||
|
||||
type WordLinks []WordLink
|
||||
type WordLinks map[WordLink]bool
|
||||
|
|
Loading…
Reference in New Issue