-
Notifications
You must be signed in to change notification settings - Fork 0
/
twitter_search.py
188 lines (168 loc) · 7.02 KB
/
twitter_search.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import argparse
from urllib.parse import urlparse
import urllib
import csv
import tweepy
# URL CLEANUP
def url_fix(s, charset='utf-8'):
if isinstance(s, unicode):
s = s.encode(charset, 'ignore')
scheme, netloc, path, qs, anchor = urlparse.urlsplit(s)
path = urllib.quote(path, '/%')
qs = urllib.quote_plus(qs, ':&=')
return urlparse.urlunsplit((scheme, netloc, path, qs, anchor))
# COMMAND PARSER
def tw_parser():
global qw, ge, l, t, c, d
# USE EXAMPLES:
# =-=-=-=-=-=-=
# % twsearch <search term> --- searches term
# % twsearch <search term> -g sf --- searches term in SF geographic box <DEFAULT = none>
# % twsearch <search term> -l en --- searches term with lang=en (English) <DEFAULT = en>
# % twsearch <search term> -t {m,r,p} --- searches term of type: mixed, recent, or popular <DEFAULT = recent>
# % twsearch <search term> -c 12 --- searches term and returns 12 tweets (count=12) <DEFAULT = 1>
# % twsearch <search term> -o {ca, tx, id, co, rtc) --- searches term and sets output options <DEFAULT = ca, tx>
# Parse the command
parser = argparse.ArgumentParser(description='Twitter Search')
parser.add_argument(action='store', dest='query', help='Search term string')
parser.add_argument('-g', action='store', dest='loca', help='Location (lo, nyl, nym, nyu, dc, sf, nb')
parser.add_argument('-l', action='store', dest='l', help='Language (en = English, fr = French, etc...)')
parser.add_argument('-t', action='store', dest='t', help='Search type: mixed, recent, or popular')
parser.add_argument('-c', action='store', dest='c', help='Tweet count (must be <50)')
args = parser.parse_args()
qw = args.query # Actual query word(s)
ge = ''
# Location
loca = args.loca
if (not(loca in ('lo', 'nyl', 'nym', 'nyu', 'dc', 'sf', 'nb')) and (loca)):
print ("WARNING: Location must be one of these: lo, nyl, nym, nyu, dc, sf, nb")
exit()
if loca:
ge = locords[loca]
# Language
l = args.l
if (not l):
l = "en"
if (not(l in ('en','fr','es','po','ko', 'ar'))):
print ("WARNING: Languages currently supported are: en (English), fr (French), es (Spanish), po (Portuguese), ko (Korean), ar (Arabic)")
exit()
# Tweet type
t = args.t
if (not t):
t = "recent"
if (not(t in ('mixed','recent','popular'))):
print ("WARNING: Search type must be one of: (m)ixed, (r)ecent, or (p)opular")
exit()
# Tweet count
if args.c:
c = int(args.c)
if (c > cmax):
print ("Resetting count to ",cmax," (maximum allowed)")
c = cmax
if (not (c) or (c < 1)):
c = 1
if not(args.c):
c = 1
print ("Query: %s, Location: %s, Language: %s, Search type: %s, Count: %s" %(qw,ge,l,t,c))
# AUTHENTICATION (OAuth)
def tw_oauth(authfile):
with open(authfile, "r") as f:
ak = f.readlines()
f.close()
auth1 = tweepy.auth.OAuthHandler(ak[0].replace("\n",""), ak[1].replace("\n",""))
auth1.set_access_token(ak[2].replace("\n",""), ak[3].replace("\n",""))
return tweepy.API(auth1)
def tw_search_json(query, cnt=5):
authfile = './auth.k'
api = tw_oauth(authfile)
results = {}
meta = {
'username': 'text',
'usersince': 'date',
'followers': 'numeric',
'friends': 'numeric',
'age':'numeric',
'authorid': 'text',
'authorloc': 'geo',
'geoenable': 'boolean',
'source': 'text'
}
data = []
for tweet in tweepy.Cursor(api.search, q=query, count=cnt,lang="en").items():
dTwt = {}
dTwt['username'] = tweet.author.name
dTwt['usersince'] = tweet.author.created_at #author/user profile creation date
dTwt['followers'] = tweet.author.followers_count #number of author/user followers (inlink)
dTwt['friends'] = tweet.author.friends_count #number of author/user friends (outlink)
dTwt['age'] = tweet.author.age
dTwt['authorid'] = tweet.author.id #author/user ID#
dTwt['authorloc'] = tweet.author.location #author/user location
dTwt['geoenable'] = tweet.author.geo_enabled #is author/user account geo enabled?
dTwt['source'] = tweet.source #platform source for tweet
data.append(dTwt)
results['meta'] = meta
results['data'] = data
return results
# TWEEPY SEARCH FUNCTION
def tw_search(api):
counter = 0
# Open/Create a file to append data
csvFile = open('tweets.csv','w')
#Use csv Writer
csvWriter = csv.writer(csvFile)
csvWriter.writerow(["created", "text", "retwc", "hashtag", "followers", "friends"])
for tweet in tweepy.Cursor(api.search,q = qw+"-filter:retweets",g = ge,lang = l,result_type = t,count = c).items():
#TWEET INFO
created = tweet.created_at #tweet created
text = tweet.text #tweet text
tweet_id = tweet.id #tweet ID# (not author ID#)
cords = tweet.coordinates #geographic co-ordinates
retwc = tweet.retweet_count #re-tweet count
try:
hashtag = tweet.entities[u'hashtags'][0][u'text'] #hashtags used
except:
hashtag = "None"
try:
rawurl = tweet.entities[u'urls'][0][u'url'] #URLs used
urls = url_fix(rawurl)
except:
urls = "None"
#AUTHOR INFO
username = tweet.author.name #author/user name
usersince = tweet.author.created_at #author/user profile creation date
followers = tweet.author.followers_count #number of author/user followers (inlink)
friends = tweet.author.friends_count #number of author/user friends (outlink)
age = tweet.author.age
authorid = tweet.author.id #author/user ID#
authorloc = tweet.author.location #author/user location
#TECHNOLOGY INFO
geoenable = tweet.author.geo_enabled #is author/user account geo enabled?
source = tweet.source #platform source for tweet
# Dongho 03/28/16
if(age>18):
csvWriter.writerow([created, str(text).encode("utf-8"), retwc, hashtag, followers, friends])
counter = counter +1
if (counter == c):
break
csvFile.close()
# MAIN ROUTINE
def main():
global api, cmax, locords
# Geo-coordinates of five metropolitan areas
# London, NYC (lower, middle, upper), Wash DC, San Francisco, New Brunswick (NJ)
locords = {'lo': '0, 51.503, 20km',
'nyl': '-74, 40.73, 2mi',
'nym': '-74, 40.74, 2mi',
'nyu': '-73.96, 40.78, 2mi',
'dc': '-77.04, 38.91, 2mi',
'sf': '-122.45, 37.74, 5km',
'nb': '-74.45, 40.49, 2mi'}
# Maximum allowed tweet count (note: Twitter sets this to ~180 per 15 minutes)
cmax = 10000
# OAuth key file
authfile = './auth.k'
tw_parser()
api = tw_oauth(authfile)
tw_search(api)
if __name__ == "__main__":
main()