github.com/n0madic/twitter-scraper@v0.0.0-20231104223941-296710769dd8/tweets_test.go (about) 1 package twitterscraper_test 2 3 import ( 4 "context" 5 "testing" 6 "time" 7 8 "github.com/google/go-cmp/cmp" 9 "github.com/google/go-cmp/cmp/cmpopts" 10 twitterscraper "github.com/n0madic/twitter-scraper" 11 ) 12 13 var cmpOptions = cmp.Options{ 14 cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Likes"), 15 cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Replies"), 16 cmpopts.IgnoreFields(twitterscraper.Tweet{}, "Retweets"), 17 } 18 19 func TestGetTweets(t *testing.T) { 20 count := 0 21 maxTweetsNbr := 300 22 dupcheck := make(map[string]bool) 23 scraper := twitterscraper.New() 24 err := scraper.LoginOpenAccount() 25 if err != nil { 26 t.Fatalf("LoginOpenAccount() error = %v", err) 27 } 28 for tweet := range scraper.GetTweets(context.Background(), "Twitter", maxTweetsNbr) { 29 if tweet.Error != nil { 30 t.Error(tweet.Error) 31 } else { 32 count++ 33 if tweet.ID == "" { 34 t.Error("Expected tweet ID is empty") 35 } else { 36 if dupcheck[tweet.ID] { 37 t.Errorf("Detect duplicated tweet ID: %s", tweet.ID) 38 } else { 39 dupcheck[tweet.ID] = true 40 } 41 } 42 if tweet.UserID == "" { 43 t.Error("Expected tweet UserID is empty") 44 } 45 if tweet.Username == "" { 46 t.Error("Expected tweet Username is empty") 47 } 48 if tweet.PermanentURL == "" { 49 t.Error("Expected tweet PermanentURL is empty") 50 } 51 if tweet.Text == "" { 52 t.Error("Expected tweet Text is empty") 53 } 54 if tweet.TimeParsed.IsZero() { 55 t.Error("Expected tweet TimeParsed is zero") 56 } 57 if tweet.Timestamp == 0 { 58 t.Error("Expected tweet Timestamp is greater than zero") 59 } 60 for _, video := range tweet.Videos { 61 if video.ID == "" { 62 t.Error("Expected tweet video ID is empty") 63 } 64 if video.Preview == "" { 65 t.Error("Expected tweet video Preview is empty") 66 } 67 if video.URL == "" { 68 t.Error("Expected tweet video URL is empty") 69 } 70 } 71 } 72 } 73 if count != maxTweetsNbr { 74 t.Errorf("Expected tweets count=%v, got: %v", maxTweetsNbr, count) 75 } 76 } 77 78 func assertGetTweet(t *testing.T, expectedTweet *twitterscraper.Tweet) { 79 actualTweet, err := testScraper.GetTweet(expectedTweet.ID) 80 if err != nil { 81 t.Error(err) 82 } else if diff := cmp.Diff(expectedTweet, actualTweet, cmpOptions...); diff != "" { 83 t.Error("Resulting tweet does not match the sample", diff) 84 } 85 } 86 87 func TestGetTweetWithVideo(t *testing.T) { 88 expectedTweet := twitterscraper.Tweet{ 89 ConversationID: "1328684389388185600", 90 HTML: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. <br><br>We have a place for that now—Fleets! <br><br>Rolling out to everyone starting today. <br><a href=\"https://t.co/auQAHXZMfH\"><img src=\"https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg\"/></a>", 91 ID: "1328684389388185600", 92 Name: "Twitter", 93 PermanentURL: "https://twitter.com/Twitter/status/1328684389388185600", 94 Photos: nil, 95 Text: "That thing you didn’t Tweet but wanted to but didn’t but got so close but then were like nah. \n\nWe have a place for that now—Fleets! \n\nRolling out to everyone starting today. https://t.co/auQAHXZMfH", 96 TimeParsed: time.Date(2020, 11, 17, 13, 0, 18, 0, time.FixedZone("UTC", 0)), 97 Timestamp: 1605618018, 98 UserID: "783214", 99 Username: "Twitter", 100 Videos: []twitterscraper.Video{{ 101 ID: "1328684333599756289", 102 Preview: "https://pbs.twimg.com/amplify_video_thumb/1328684333599756289/img/cP5KwbIXbGunNSBy.jpg", 103 URL: "https://video.twimg.com/amplify_video/1328684333599756289/vid/960x720/PcL8yv8KhgQ48Qpt.mp4?tag=13", 104 }}, 105 } 106 assertGetTweet(t, &expectedTweet) 107 } 108 109 func TestGetTweetWithMultiplePhotos(t *testing.T) { 110 expectedTweet := twitterscraper.Tweet{ 111 ConversationID: "1390026628957417473", 112 HTML: `no bird too tall, no crop too short<br><br>introducing bigger and better images on iOS and Android, now available to everyone <br><a href="https://t.co/2buHfhfRAx"><img src="https://pbs.twimg.com/media/E0pd2L2XEAQ_gnn.jpg"/></a><br><img src="https://pbs.twimg.com/media/E0pd2hPXoAY9-TZ.jpg"/>`, 113 ID: "1390026628957417473", 114 Name: "Twitter", 115 PermanentURL: "https://twitter.com/Twitter/status/1390026628957417473", 116 Photos: []twitterscraper.Photo{ 117 {ID: "1390026620472332292", URL: "https://pbs.twimg.com/media/E0pd2L2XEAQ_gnn.jpg"}, 118 {ID: "1390026626214371334", URL: "https://pbs.twimg.com/media/E0pd2hPXoAY9-TZ.jpg"}, 119 }, 120 Text: "no bird too tall, no crop too short\n\nintroducing bigger and better images on iOS and Android, now available to everyone https://t.co/2buHfhfRAx", 121 TimeParsed: time.Date(2021, 5, 5, 19, 32, 28, 0, time.FixedZone("UTC", 0)), 122 Timestamp: 1620243148, 123 UserID: "783214", 124 Username: "Twitter", 125 } 126 assertGetTweet(t, &expectedTweet) 127 } 128 129 func TestGetTweetWithGIF(t *testing.T) { 130 if skipAuthTest { 131 t.Skip("Skipping test due to environment variable") 132 } 133 expectedTweet := twitterscraper.Tweet{ 134 ConversationID: "1288540609310056450", 135 GIFs: []twitterscraper.GIF{ 136 { 137 ID: "1288540582768517123", 138 Preview: "https://pbs.twimg.com/tweet_video_thumb/EeHQ1UKXoAMVxWB.jpg", 139 URL: "https://video.twimg.com/tweet_video/EeHQ1UKXoAMVxWB.mp4", 140 }, 141 }, 142 Hashtags: []string{"CountdownToMars"}, 143 HTML: `Like for liftoff! <a href="https://twitter.com/hashtag/CountdownToMars">#CountdownToMars</a> <br><a href="https://t.co/yLe331pHfY"><img src="https://pbs.twimg.com/tweet_video_thumb/EeHQ1UKXoAMVxWB.jpg"/></a>`, 144 ID: "1288540609310056450", 145 Name: "Twitter", 146 PermanentURL: "https://twitter.com/Twitter/status/1288540609310056450", 147 Text: "Like for liftoff! #CountdownToMars https://t.co/yLe331pHfY", 148 TimeParsed: time.Date(2020, 7, 29, 18, 23, 15, 0, time.FixedZone("UTC", 0)), 149 Timestamp: 1596046995, 150 UserID: "783214", 151 Username: "Twitter", 152 } 153 assertGetTweet(t, &expectedTweet) 154 } 155 156 func TestGetTweetWithPhotoAndGIF(t *testing.T) { 157 if skipAuthTest { 158 t.Skip("Skipping test due to environment variable") 159 } 160 expectedTweet := twitterscraper.Tweet{ 161 ConversationID: "1580661436132757506", 162 GIFs: []twitterscraper.GIF{ 163 { 164 ID: "1580661428335382531", 165 Preview: "https://pbs.twimg.com/tweet_video_thumb/Fe-jMcIXkAMXK_W.jpg", 166 URL: "https://video.twimg.com/tweet_video/Fe-jMcIXkAMXK_W.mp4", 167 }, 168 }, 169 HTML: `a hit Tweet <br><a href="https://t.co/2C7cah4KzW"><img src="https://pbs.twimg.com/media/Fe-jMcGWQAAFWoG.jpg"/></a><br><img src="https://pbs.twimg.com/tweet_video_thumb/Fe-jMcIXkAMXK_W.jpg"/>`, 170 ID: "1580661436132757506", 171 Name: "Twitter", 172 PermanentURL: "https://twitter.com/Twitter/status/1580661436132757506", 173 Photos: []twitterscraper.Photo{{ID: "1580661428326907904", URL: "https://pbs.twimg.com/media/Fe-jMcGWQAAFWoG.jpg"}}, 174 Text: "a hit Tweet https://t.co/2C7cah4KzW", 175 TimeParsed: time.Date(2022, 10, 13, 20, 47, 8, 0, time.FixedZone("UTC", 0)), 176 Timestamp: 1665694028, 177 UserID: "783214", 178 Username: "Twitter", 179 } 180 assertGetTweet(t, &expectedTweet) 181 } 182 183 func TestTweetMentions(t *testing.T) { 184 sample := []twitterscraper.Mention{{ 185 ID: "7018222", 186 Username: "davidmcraney", 187 Name: "David McRaney", 188 }} 189 tweet, err := testScraper.GetTweet("1554522888904101890") 190 if err != nil { 191 t.Error(err) 192 } else { 193 if diff := cmp.Diff(sample, tweet.Mentions, cmpOptions...); diff != "" { 194 t.Error("Resulting tweet does not match the sample", diff) 195 } 196 } 197 } 198 199 func TestQuotedAndReply(t *testing.T) { 200 sample := &twitterscraper.Tweet{ 201 ConversationID: "1237110546383724547", 202 HTML: "The Easiest Problem Everyone Gets Wrong <br><br>[new video] --> <a href=\"https://youtu.be/ytfCdqWhmdg\">https://t.co/YdaeDYmPAU</a> <br><a href=\"https://t.co/iKu4Xs6o2V\"><img src=\"https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg\"/></a>", 203 ID: "1237110546383724547", 204 Likes: 485, 205 Name: "Vsauce2", 206 PermanentURL: "https://twitter.com/VsauceTwo/status/1237110546383724547", 207 Photos: []twitterscraper.Photo{{ 208 ID: "1237110473486729218", 209 URL: "https://pbs.twimg.com/media/ESsZa9AXgAIAYnF.jpg", 210 }}, 211 Replies: 12, 212 Retweets: 18, 213 Text: "The Easiest Problem Everyone Gets Wrong \n\n[new video] --> https://t.co/YdaeDYmPAU https://t.co/iKu4Xs6o2V", 214 TimeParsed: time.Date(2020, 0o3, 9, 20, 18, 33, 0, time.FixedZone("UTC", 0)), 215 Timestamp: 1583785113, 216 URLs: []string{"https://youtu.be/ytfCdqWhmdg"}, 217 UserID: "978944851", 218 Username: "VsauceTwo", 219 } 220 tweet, err := testScraper.GetTweet("1237110897597976576") 221 if err != nil { 222 t.Error(err) 223 } else { 224 if !tweet.IsQuoted { 225 t.Error("IsQuoted must be True") 226 } 227 if diff := cmp.Diff(sample, tweet.QuotedStatus, cmpOptions...); diff != "" { 228 t.Error("Resulting quote does not match the sample", diff) 229 } 230 } 231 tweet, err = testScraper.GetTweet("1237111868445134850") 232 if err != nil { 233 t.Error(err) 234 } else { 235 if !tweet.IsReply { 236 t.Error("IsReply must be True") 237 } 238 if diff := cmp.Diff(sample, tweet.InReplyToStatus, cmpOptions...); diff != "" { 239 t.Error("Resulting reply does not match the sample", diff) 240 } 241 } 242 243 } 244 func TestRetweet(t *testing.T) { 245 sample := &twitterscraper.Tweet{ 246 ConversationID: "1359151057872580612", 247 HTML: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", 248 ID: "1359151057872580612", 249 IsSelfThread: false, 250 Likes: 6683, 251 Name: "Twitter Together", 252 PermanentURL: "https://twitter.com/TwitterTogether/status/1359151057872580612", 253 Replies: 456, 254 Retweets: 1495, 255 Text: "We’ve seen an increase in attacks against Asian communities and individuals around the world. It’s important to know that this isn’t new; throughout history, Asians have experienced violence and exclusion. However, their diverse lived experiences have largely been overlooked.", 256 TimeParsed: time.Date(2021, 02, 9, 14, 43, 58, 0, time.FixedZone("UTC", 0)), 257 Timestamp: 1612881838, 258 UserID: "773578328498372608", 259 Username: "TwitterTogether", 260 } 261 tweet, err := testScraper.GetTweet("1362849141248974853") 262 if err != nil { 263 t.Error(err) 264 } else { 265 if !tweet.IsRetweet { 266 t.Error("IsRetweet must be True") 267 } 268 if diff := cmp.Diff(sample, tweet.RetweetedStatus, cmpOptions...); diff != "" { 269 t.Error("Resulting retweet does not match the sample", diff) 270 } 271 } 272 } 273 274 func TestTweetViews(t *testing.T) { 275 sample := &twitterscraper.Tweet{ 276 HTML: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.", 277 ID: "1606055187348688896", 278 Likes: 2839, 279 Name: "Twitter Support", 280 PermanentURL: "https://twitter.com/TwitterSupport/status/1606055187348688896", 281 Replies: 3427, 282 Retweets: 783, 283 Text: "Replies and likes don’t tell the whole story. We’re making it easier to tell *just* how many people have seen your Tweets with the addition of view counts, shown right next to likes. Now on iOS and Android, web coming soon.", 284 TimeParsed: time.Date(2022, 12, 22, 22, 32, 50, 0, time.FixedZone("UTC", 0)), 285 Timestamp: 1612881838, 286 UserID: "17874544", 287 Username: "TwitterSupport", 288 Views: 3189278, 289 } 290 tweet, err := testScraper.GetTweet("1606055187348688896") 291 if err != nil { 292 t.Error(err) 293 } else { 294 if tweet.Views < sample.Views { 295 t.Error("Views must be greater than or equal to the sample") 296 } 297 } 298 } 299 300 func TestTweetThread(t *testing.T) { 301 if skipAuthTest { 302 t.Skip("Skipping test due to environment variable") 303 } 304 tweet, err := testScraper.GetTweet("1665602315745673217") 305 if err != nil { 306 t.Fatal(err) 307 } else { 308 if !tweet.IsSelfThread { 309 t.Error("IsSelfThread must be True") 310 } 311 if len(tweet.Thread) != 7 { 312 t.Error("Thread length must be 7") 313 } 314 } 315 }