Repository navigation
Expand file tree
/
Copy pathbuildarff.py
More file actions
417 lines (376 loc) · 13.1 KB
/
Copy pathbuildarff.py
File metadata and controls
417 lines (376 loc) · 13.1 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
import sys
import re
from functools import partial
CLASSES = ['0', '4']
def init_arff(f):
'''
Initialize the .arff file f. That is, write the relation name
and enumerate the attributes.
'''
f.write('@relation {}\n\n'.format(RELATION_NAME))
for feature in FEATURES:
f.write('@attribute {} numeric\n'.format(feature))
f.write('@attribute emotion {' + ','.join(CLASSES) + '}\n\n')
f.write('@data\n')
def all_data_to_arff(f, data):
'''
Write all of data to f in accordance with .arff specifications.
'''
for d in data:
write_arff_data_point(f, d)
def write_arff_data_point(f, d):
'''
Write the single datapoint d to f in .arff format.
'''
res = []
for feature in FEATURES:
func = FEATURE_FUNCS[feature]
res += [str(func(d))]
res += [str(get_class(d))]
f.write(','.join(res) + '\n')
def load_tweets(input_file_name, max_per_class):
'''
Return a list of at most max_per_class normalized tweets
for every class in CLASSES, all of which are stored in
the file named input_file_name.
If max_per_class is -1, return all tweets for each class in CLASSES.
'''
if max_per_class == -1:
data = load_all_tweets(input_file_name)
else:
data = load_n_tweets(input_file_name, max_per_class)
return data
def load_n_tweets(input_file_name, n):
'''
Return n normalized tweets per class in a list.
'''
data = []
for cls in CLASSES:
f = open(input_file_name, 'r')
data += load_n_class_tweets(f, n, cls)
f.close()
return data
def load_n_class_tweets(f, n, cls):
'''
Return n normalized tweets - stored in file f - of class cls.
'''
data = []
L = f.readlines()
curr, new = (L[0], False) if is_demarcation(L[0], cls) else ('', True)
for line in L[1:]:
if is_demarcation(line, cls):
curr = line
new = False
elif not new and not is_demarcation(line):
curr += line
elif is_demarcation(line) and curr:
data += [curr]
curr = ''
new = True
if len(data) == n:
break
return data
def load_all_tweets(input_file_name):
'''
Return every normalized tweet stored in the file
input_file_name in a list.
'''
data = []
f = open(input_file_name, 'r')
L = f.readlines()
curr = L[0]
for line in L[1:]:
if is_demarcation(line):
data += [curr]
curr = line
else:
curr += line
data += [curr]
f.close()
return data
def is_demarcation(s, c=''.join(CLASSES)):
'''
Return True iff s is a tweet demarcation of the form
<A=c>, where c is the class of that tweet.
By default, we check for any class.
'''
return True if re.match('<A=[{}]>'.format(c), s.rstrip('\n')) else False
def is_punctuation(s):
'''
Return True iff s is a punctuation token.
'''
return True if re.match('[,:;]|[\.!?()$"`' + "']", s) else False
def get_class(tweet):
'''
Return the class of tweet.
'''
# as follows:
demarcation = tweet.split('\n')[0]
cls = [int(i) for i in demarcation if i.isdigit()][0]
return cls
def as_sentences(tweet):
'''
Return the input tweet as a list of sentences,
omitting the demarcation.
'''
# Upon splitting on newlines, the first element
# of the resulting list will be the tweet's
# demarcation and the last element will be the
# empty string. All other elements are sentences.
return tweet.split('\n')[1:-1]
def sentence_to_tags(sentence):
'''
Return a list of tags corresponding to the tagged tokens
in sentence.
'''
split_tags = map(lambda s: s.split('/')[1], sentence.split(' '))
return filter(None, split_tags)
def sentence_to_tokens(sentence):
'''
Return a list of tokens corresponding to the tagged tokens
in sentence.
'''
split_tokens = map(lambda s: s.split('/')[0], sentence.split(' '))
return filter(None, split_tokens)
def sentence_no_tags(sentence):
'''
Return a string of the input sentence with tags stripped
from tokens.
'''
return ' '.join(sentence_to_tokens(sentence))
def count_sentences(tweet):
'''
Return the number of sentences in tweet.
'''
return len(as_sentences(tweet))
def avg_token_length(tweet):
'''
Return the average length of tokens in tweet,
measured by number of characters.
'''
num_tokens = 0
total = 0
for sentence in as_sentences(tweet):
tokens = sentence_to_tokens(sentence)
lengths = map(lambda s: 0 if is_punctuation(s) else len(s), tokens)
num_tokens += len(lengths)
total += sum(lengths)
return total / num_tokens
def avg_sentence_length(tweet):
'''
Return the average length of sentences in tweet,
measured by number of tokens
'''
num_sentences = count_sentences(tweet)
total = 0
for sentence in as_sentences(tweet):
tokens = sentence_to_tokens(sentence)
total += len(tokens)
return total / num_sentences
def count_pronoun(nth_person, tweet):
'''
Return the number of occurences in tweet of nth_person
pronouns.
'''
f = open(WORDLISTS_DIR + '/{}-person'.format(nth_person), 'r')
pronouns = [s.rstrip('\n') for s in f.readlines()]
f.close()
count = 0
for sentence in as_sentences(tweet):
tokens = sentence_to_tokens(sentence)
token_to_pronoun = map(lambda s: 1
if s.lower() in pronouns
else 0, tokens)
count += sum(token_to_pronoun)
return count
def count_conjunctions(tweet):
'''
Return the number of occurences in tweet of the following:
'and', 'but', 'for', 'nor', 'or', 'so', 'yet'
'''
count = 0
for sentence in as_sentences(tweet):
tags = sentence_to_tags(sentence)
count += tags.count('CC')
return count
def count_verbs(tense, tweet):
'''
Return the number of occurences in tweet of
tense-tense verbs in tweet.
'''
tense_regex = VERB_TENSE_REGEX[tense]
count = 0
for sentence in as_sentences(tweet):
count += regex_count(sentence, tense_regex)
return count
def regex_count(sentence, regex):
'''
Return the number of times regex is found in sentence.
'''
r = re.compile(regex, re.IGNORECASE)
return len(r.findall(sentence))
def count_p_nouns(tweet):
'''
Return the number of occurences in tweet of proper nouns.
'''
count = 0
for sentence in as_sentences(tweet):
tags = sentence_to_tags(sentence)
count += tags.count('NNP') + tags.count('NNPS')
return count
def count_c_nouns(tweet):
'''
Return the number of occurences in tweet of common nouns
'''
count = 0
for sentence in as_sentences(tweet):
tags = sentence_to_tags(sentence)
count += tags.count('NN') + tags.count('NNS')
return count
def count_adverbs(tweet):
'''
Return the number of occurences in tweet of adverbs.
Adverbs are tagged RB, RBR, or RBS.
'''
count = 0
for sentence in as_sentences(tweet):
tags = sentence_to_tags(sentence)
count += tags.count('RB') + tags.count('RBR') + tags.count('RBS')
return count
def count_wh_words(tweet):
'''
Return the number of occurences in tweet of wh-words,
i.e. words tagged with WDT / WP / WP$ / WRB.
'''
count = 0
for sentence in as_sentences(tweet):
tags = sentence_to_tags(sentence)
count += tags.count('WDT') + tags.count('WP')
count += tags.count('WP$') + tags.count('WRB')
return count
def count_slang(tweet):
'''
Return the number of occurences in tweet of slang words.
'''
count = 0
f = open(WORDLISTS_DIR + '/Slang', 'r')
slang_words = [s.rstrip('\n') for s in f.readlines()]
f.close()
for sentence in as_sentences(tweet):
tokens = sentence_to_tokens(sentence)
token_to_slang = map(lambda s: 1
if s.lower() in slang_words
else 0, tokens)
count += sum(token_to_slang)
return count
def count_uppercase(tweet):
'''
Return the number of occurences of uppercase words of
length greater than or equal to 2.
'''
count = 0
for sentence in as_sentences(tweet):
tokens = sentence_to_tokens(sentence)
token_to_upper = map(lambda s: 1
if len(s) > 1 and s.isupper()
else 0, tokens)
count += sum(token_to_upper)
return count
def count_punctuations(tweet):
'''
Return the number of occurences of the following
characters:
, (commas)
: | ; (colons or semi colons)
- (dashes)
( (right parenthesis)
) (left parenthesis)
... (ellipses)
'''
punc_to_count = {}
for punc in PUNCTUATIONS:
punc_to_count[punc] = count_punctuation(tweet, punc)
return punc_to_count
def count_punctuation(punc, tweet):
'''
Return the number of occurences in tweet of punc.
Multiple punctuations are considered single tokens,
so we count how many matches of the regular expression
punc+ we find in tweet.
'''
count = 0
for sentence in as_sentences(tweet):
tokens_no_tags = sentence_no_tags(sentence)
count += len(re.findall('{}+'.format(punc), tokens_no_tags))
return count
RELATION_NAME = 'twit_classification'
######################## EXTENDING FEATURES ##############################
#
# If we ever want to extract a new feature, we need only carry out
# three steps:
#
# 1. Write a function that returns a numeric value
# representing the value of that feature. The signature of
# the function must be of the format
# def <feature_function>(*args, tweet)
# where *args is a sequence of optional arguments (0 or more)
# and tweet is the input tweet in .twtt format.
#
# 2. Add the name of the feature to the global list FEATURES.
#
# 3. Add a mapping to the global dictionary FEATURE_FUNCS that is
# of the form
# <feature_name>: <feature_function>
# If the function calculates the feature value in the same way
# for related features, the mapping will be of the form
# <feature_name>: partial(<feature_function>, *args)
# which is a partial function application depending on the
# specific sub-feature. Note that for multiple optional arguments,
# we must apply partial as many times as there are arguments.
#
##########################################################################
FEATURES = ['1st_person_pro', '2nd_person_pro', '3rd_person_pro',
'conjunctions', 'past', 'future', 'commas', '(semi)colons',
'dashes', 'parentheses', 'ellipses', 'common_nouns',
'proper_nouns', 'adverbs', 'wh_words', 'slang', 'uppercase',
'avg_sentence_len', 'avg_token_len', 'num_sentences']
FEATURE_FUNCS = {'1st_person_pro' : partial(count_pronoun, 'First'),
'2nd_person_pro' : partial(count_pronoun, 'Second'),
'3rd_person_pro' : partial(count_pronoun, 'Third'),
'conjunctions' : count_conjunctions,
'past' : partial(count_verbs, 'past'),
'future' : partial(count_verbs, 'future'),
'commas' : partial(count_punctuation, ','),
'(semi)colons' : partial(count_punctuation, '(:|;)'),
'dashes' : partial(count_punctuation, '-'),
'parentheses' : partial(count_punctuation, '(\(|\))'),
'ellipses' : partial(count_punctuation, '(\.\.\.)'),
'common_nouns' : count_c_nouns,
'proper_nouns' : count_p_nouns,
'adverbs' : count_adverbs,
'wh_words' : count_wh_words,
'slang' : count_slang,
'uppercase' : count_uppercase,
'avg_sentence_len' : avg_sentence_length,
'avg_token_len' : avg_token_length,
'num_sentences' : count_sentences}
PAST_TAGS = "(/PRP\$?|/VBD|/VBG|/VBN)"
FUTURE_TAGS = "(/PRP\$|/VB|/VBP|/VBZ|/MD)"
PAST_FIXES = "(has|had|have|were|was|did)?"
FUTURE_FIXES = "('nt|'ll|will|won't|gonna|going\sto)?"
PAST_REGEX = PAST_FIXES + PAST_TAGS + '\s?[a-z]*' + PAST_TAGS
FUTURE_REGEX = FUTURE_FIXES + FUTURE_TAGS + '\s?[a-z]*' + FUTURE_TAGS
VERB_TENSE_REGEX = {'past': PAST_REGEX, 'future': FUTURE_REGEX}
WORDLISTS_DIR = '/u/cs401/Wordlists'
if __name__ == "__main__":
args = sys.argv
input_file_name = args[1] # eg train.twt
output_file_name = args[2] # eg train.arff
if len(args) == 4:
max_per_class = int(args[3])
else:
max_per_class = -1
data = load_tweets(input_file_name, max_per_class)
with open(output_file_name, 'w+') as f:
init_arff(f)
all_data_to_arff(f, data)