Repository navigation
Expand file tree
/
Copy pathtwtt.py
More file actions
296 lines (258 loc) · 9.25 KB
/
Copy pathtwtt.py
File metadata and controls
296 lines (258 loc) · 9.25 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
# This file parses the raw csv data into normalized form.
import NLPlib
import sys
import csv
import re
import codecs
import HTMLParser
import io
# WARNING: Change this before submitting.
ABBR_FILE = '/u/cs401/Wordlists/abbrev.english'
TAGGER = NLPlib.NLPlib()
CLASSES = [0, 4]
CLASS_INDICES = {0: 0, 4: 800000}
NUM_TRAIN = 11000
with open(ABBR_FILE) as f:
ABBREVIATIONS = map(lambda x: x.rstrip('\n'), f.readlines())
def strip_html(tweet): # Step 1 of part 1.
"""
Return the input tweet with all html tags and attributes
removed.
input:
tweet - a string representing a tweet.
output:
tweet - a string representing a tweet.
"""
mo = re.search("[\s]*<[^>]+>" , tweet)
while mo:
tweet = tweet[:mo.start()] + tweet[mo.end():]
mo = re.search("[\s]*<[^>]+>" , tweet)
return tweet
def html_char_to_ascii(tweet): # Step 2 of part 1.
"""
Return the input tweet with all HTML character codes
replaced with their corresponding ASCII characters.
input:
tweet - a string representing a tweet.
ouput:
tweet - a string representing a tweet.
"""
tweet = ''.join([c if ord(c) < 128 else '' for c in tweet])
h = HTMLParser.HTMLParser()
return h.unescape(tweet)
def strip_urls(tweet): # Step 3 of part 1.
"""
Return the input tweet with all URLs removed.
A URL is any substring beginning with one of
the following:
- 'http://'
- 'https://'
- 'www.'
input:
tweet - a string representing a tweet.
output:
tweet - a string representing a tweet.
"""
mo = re.search("[\s]*([Hh]ttps?|www)[^\s]*[\s]*", tweet)
while mo:
tweet = tweet[:mo.start()] + tweet[mo.end()-1:]
mo = re.search("[\s]*([Hh]ttps?|www)[^\s]*[\s]*", tweet)
return tweet
def strip_twitter_chars(tweet): # Step 4 of part 1.
"""
Return the input tweet with Twitter-specific characters removed.
That is, username tags (@) and hash tags (#).
input:
tweet - a string representing a tweet.
output:
tweet - a string representing a tweet.
"""
mo = re.search(r"[#@]" , tweet)
while mo:
tweet = tweet[:mo.start()] + tweet[mo.end():]
mo = re.search(r"[#@]" , tweet)
return tweet
def split_sentences(tweet): # Step 5 of part 1.
"""
Return the input tweet with all distinct sentences separated by
a newline character.
input:
tweet - a string representing a tweet.
ouput:
split_tweet - a string representing a tweet, where each
distinct sentence is on it's own line.
"""
split_tweet = ''
mo = re.search("[\.!?]+", tweet)
i = 0
while mo:
n = len(tweet)
if mo.end() < n and tweet[mo.end()] in ["'", '"']:
split_tweet += tweet[i:mo.end() + 1] + '\n'
i = 1
elif tweet[mo.end() - 1] in ['!', '?']:
split_tweet += tweet[i:mo.end()] + '\n'
i = 0
elif tweet[mo.end() - 1] == '.':
if is_abbreviation(tweet, mo.end() - 1):
split_tweet += tweet[i:mo.end()+1]
else:
split_tweet += tweet[i:mo.end()] + '\n'
i = 0
tweet = tweet[mo.end() + 1:]
mo = re.search("[\.!?]+", tweet)
return (split_tweet + tweet).rstrip('\n')
def is_abbreviation(tweet, i):
"""
Return True iff the word terminated by the period at index i is
an abbreviation.
input:
tweet - a string representing a tweet.
i - the index of a period in tweet.
output:
True - the word terminated at index i is an abbreviation.
False - otherwise.
"""
word = ''
while i >= 0:
word += tweet[i]
i -= 1
if tweet[i] == ' ':
break
return word[::-1] in ABBREVIATIONS
def space_tokens(tweet): # Step 7 of part 1.
"""
Return the input tweet with all distinct token separated by a space.
input:
tweet - a string representing a tweet.
output:
tweet - a string representing a tweet.
"""
mo = re.search("[,:;]|[\.!?]+" , tweet) # dog... asdf -> dog ... asdf
split_tweet = ''
while mo:
split_tweet += tweet[:mo.start()] + ' ' + tweet[mo.start():mo.end()]
tweet = tweet[mo.end():]
mo = re.search("[,:;]|[\.!?]+" , tweet)
return space_clitics(split_tweet + tweet)
def space_clitics(tweet):
"""
Return the input tweet with spaces between each clitic.
input:
tweet - a string where each word and punctuation mark
are separated by a space.
output:
split_tweet - a string where each word, punctuation mark,
and clitic are separated by a space.
"""
# First we match a quote followed by zero or more characters, then a space.
mo = re.search("'[^\s]*\s", tweet) # Will detect any quote marks and go to the end of the word.
split_tweet = ''
while mo:
# If the quote is not followed by an s, we split the quote. So dogs' -> dogs '
if tweet[mo.start()+1] == 't': # we grab the char preceding the apostrophe.
split_tweet += tweet[:mo.start()-1] + ' ' + tweet[mo.start()-1:mo.end()]
else:
# We split from apostrophe onward to end of word
# (this includes cases such as dogs' -> dogs ' )
# before mo.start() is where we want to insert a space
split_tweet += tweet[:mo.start()] + ' ' + tweet[mo.start():mo.end()]
tweet = tweet[mo.end():]
mo = re.search("'[^\s]*\s", tweet)
return split_tweet + tweet
def tag_tokens(tweet): # Step 8 of part 1.
"""
Return a string where space-separated tokens are each tagged
with their part-of-speech.
input:
tweet - a string representing a tweet.
output:
tagged - a string of space-separated tokens, each tagged
with their part-of-speech.
"""
tokens = tweet.split(' ')
tags = TAGGER.tag(tokens)
for i in range(0, len(tokens)):
tokens[i] = tokens[i].rstrip('\n') + '/' + tags[i]
tokens = " ".join(tokens)
return tokens
def add_demarcation(tweet, n): # Step 9 of part 1.
"""
Return the input tweet prefixed with a demarcation (on a separate line)
representing the numeric class of the tweet.
input:
tweet - a string representing a tweet.
n - the numeric class of tweet.
output:
labeled_tweet - the input tweet prefixed with it's
numeric class n demarcated on a separate line.
"""
demarcation = '<A={}>\n'.format(n)
return demarcation + tweet
def normalize_tweet(tweet_text, tweet_class):
"""
Return the input tweet in normalized form. That is:
- all HTML tags and attributes are removed
- all HTML characters code are replaced with their ASCII equivalents
- all URLs are removed
- each sentence within a tweet is on its own line
- each token, including punctuation and cltiics, is separated by spaces
- ellipsis and other kinds of multiple punctuation are not split
- each token is tagged with its part-of-speech
- each tweet is labeled with its class in a prefixing demarcation
input:
tweet - a string representing a tweet.
output:
normalized_tweet - the input tweet in normalized form.
"""
# Apply all filters here.
tweet_text = strip_html(tweet_text)
tweet_text = html_char_to_ascii(tweet_text)
tweet_text = strip_urls(tweet_text)
tweet_text = strip_twitter_chars(tweet_text)
tweet_text = split_sentences(tweet_text)
tweet_text = space_tokens(tweet_text)
newline_split_tweets = tweet_text.split('\n')
tweet_text = ''
for line in newline_split_tweets:
tweet_text += tag_tokens(line) + '\n'
tweet_text = tweet_text.rstrip('\n')
tweet_text = add_demarcation(tweet_text, tweet_class)
return tweet_text + '\n'
def partition_by_class(reader, gid, n):
'''
Return a list of a total of n data points with
equal representation for each class in CLASSES.
Data is stored in reader, which was obtained by reading
a formatted CSV. Partitions are determined by gid, which is the
group id.
'''
num_classes = len(CLASSES)
num_per_class = n // num_classes
data = []
for cls in CLASSES:
lower_i = CLASS_INDICES[cls] + (gid * num_per_class)
upper_i = CLASS_INDICES[cls] + ((gid + 1) * num_per_class)
data += reader[lower_i:upper_i]
return data
def main(tweets, output_file):
for tweet in tweets: # A tweet is a list of length 6, containing the various fields.
normalized_tweet = normalize_tweet(tweet[-1], tweet[0])
output_file.write(normalized_tweet)
if __name__ == "__main__":
args = sys.argv
argc = len(args)
input_file_name = args[1]
f_in = open(input_file_name, 'r')
reader = list(csv.reader(f_in, delimiter=','))
if argc == 4:
gid = int(args[2])
output_file_name = args[3]
data = partition_by_class(reader, gid, NUM_TRAIN)
elif argc == 3:
output_file_name = args[2]
data = reader
f_out = open(output_file_name, 'w+')
main(data, f_out)
f_in.close()
f_out.close()