Refactoring

scraper.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"gopkg.in/gcfg.v1"
 	"os"
-	"strings"
 )
 
 func main() {
@@ -21,29 +20,9 @@ func main() {
 	// Get Twitter Client
 	twitter = WorstCaptcha.NewTwitter(config)
 
-	// Do twitter search and loop
-	for _, tweet := range twitter.DoSearch() {
-		fmt.Printf("\n=================\n")
-
-		// Type assert entities to be a map
-		entities := tweet["entities"].(map[string]interface{})
-
-		// Type assert media to be a list
-		mediaList := entities["media"].([]interface{})
-
-		// Type assert the media to be a map
-		media := mediaList[0].(map[string]interface{})
-
-		// Get the link
-		imageUrl := media["media_url_https"]
-
-		// Parts of word
-		parts := strings.Split(tweet.Text(), " ")
-
-		// Build word without link
-		word := strings.Join(parts[0:len(parts)-1], " ")
-
-		fmt.Println(imageUrl)
-		fmt.Println(word)
+	// Get images
+	for wordlink, _ := range twitter.GetImages() {
+		fmt.Println("===================")
+		fmt.Println(wordlink)
 	}
 }

src/WorstCaptcha/Twitter.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"net/url"
 	"os"
+	"strings"
 )
 
 func NewTwitter(config Config) Twitter {
@@ -27,7 +28,10 @@ type Twitter struct {
 	client *twittergo.Client
 }
 
-func (t *Twitter) DoSearch() []twittergo.Tweet {
+//
+// Searching
+//
+func (t *Twitter) doSearch() []twittergo.Tweet {
 	var (
 		err     error
 		req     *http.Request
@@ -71,3 +75,41 @@ func (t *Twitter) DoSearch() []twittergo.Tweet {
 
 	return results.Statuses()
 }
+
+//
+// Get Images from twitter
+//
+func (t *Twitter) GetImages() WordLinks {
+	var wls = WordLinks{}
+
+	for _, tweet := range t.doSearch() {
+		wls[t.filterTweet(tweet)] = true
+	}
+
+	return wls
+}
+
+//
+// Convert a single tweet to a set of word and link
+//
+func (t *Twitter) filterTweet(tweet twittergo.Tweet) WordLink {
+	// Type assert entities to be a map
+	entities := tweet["entities"].(map[string]interface{})
+
+	// Type assert media to be a list
+	mediaList := entities["media"].([]interface{})
+
+	// Type assert the media to be a map
+	media := mediaList[0].(map[string]interface{})
+
+	// Get the link
+	imageUrl := media["media_url_https"].(string)
+
+	// Parts of word
+	parts := strings.Split(tweet.Text(), " ")
+
+	// Build word without link
+	word := strings.Join(parts[0:len(parts)-1], " ")
+
+	return WordLink{word, imageUrl}
+}

src/WorstCaptcha/WordLink.go

@@ -5,4 +5,4 @@ type WordLink struct {
 	Link string
 }
 
-type WordLinks []WordLink
+type WordLinks map[WordLink]bool