Repository navigation
Expand file tree
/
Copy pathibmTest.py
More file actions
238 lines (208 loc) · 7.97 KB
/
Copy pathibmTest.py
File metadata and controls
238 lines (208 loc) · 7.97 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
# ibmTest.py
#
# This file tests all 3 classifiers using the NLClassifier IBM Service
# previously created using ibmTrain.py
#
# TODO: You must fill out all of the functions in this file following
# the specifications exactly. DO NOT modify the headers of any
# functions. Doing so will cause your program to fail the autotester.
#
# You may use whatever libraries you like (as long as they are available
# on CDF). You may find json, request, or pycurl helpful.
# You may also find it helpful to reuse some of your functions from ibmTrain.py.
#
def get_classifier_ids(username,password):
# Retrieves a list of classifier ids from a NLClassifier service
# an outputfile named ibmTrain#.csv (where # is n_lines_to_extract).
#
# Inputs:
# username - username for the NLClassifier to be used, as a string
#
# password - password for the NLClassifier to be used, as a string
#
#
# Returns:
# a list of classifier ids as strings
#
# Error Handling:
# This function should throw an exception if the classifiers call fails for any reason
#
#TODO: Fill in this function
return
def assert_all_classifiers_are_available(username, password, classifier_id_list):
# Asserts all classifiers in the classifier_id_list are 'Available'
#
# Inputs:
# username - username for the NLClassifier to be used, as a string
#
# password - password for the NLClassifier to be used, as a string
#
# classifier_id_list - a list of classifier ids as strings
#
# Returns:
# None
#
# Error Handling:
# This function should throw an exception if the classifiers call fails for any reason AND
# It should throw an error if any classifier is NOT 'Available'
#
#TODO: Fill in this function
return
def classify_single_text(username,password,classifier_id,text):
# Classifies a given text using a single classifier from an NLClassifier
# service
#
# Inputs:
# username - username for the NLClassifier to be used, as a string
#
# password - password for the NLClassifier to be used, as a string
#
# classifier_id - a classifier id, as a string
#
# text - a string of text to be classified, not UTF-8 encoded
# ex. "Oh, look a tweet!"
#
# Returns:
# A "classification". Aka:
# a dictionary containing the top_class and the confidences of all the possible classes
# Format example:
# {'top_class': 'class_name',
# 'classes': [
# {'class_name': 'myclass', 'confidence': 0.999} ,
# {'class_name': 'myclass2', 'confidence': 0.001}
# ]
# }
#
# Error Handling:
# This function should throw an exception if the classify call fails for any reason
#
#TODO: Fill in this function
return
def classify_all_texts(username,password,input_csv_name):
# Classifies all texts in an input csv file using all classifiers for a given NLClassifier
# service.
#
# Inputs:
# username - username for the NLClassifier to be used, as a string
#
# password - password for the NLClassifier to be used, as a string
#
# input_csv_name - full path and name of an input csv file in the
# 6 column format of the input test/training files
#
# Returns:
# A dictionary of lists of "classifications".
# Each dictionary key is the name of a classifier.
# Each dictionary value is a list of "classifications" where a
# "classification" is in the same format as returned by
# classify_single_text.
# Each element in the main dictionary is:
# A list of dictionaries, one for each text, in order of lines in the
# input file. Each element is a dictionary containing the top_class
# and the confidences of all the possible classes (ie the same
# format as returned by classify_single_text)
# Format example:
# {‘classifiername’:
# [
# {'top_class': 'class_name',
# 'classes': [
# {'class_name': 'myclass', 'confidence': 0.999} ,
# {'class_name': 'myclass2', 'confidence': 0.001}
# ]
# },
# {'top_class': 'class_name',
# ...
# }
# ]
# , ‘classifiername2’:
# [
# …
# ]
# …
# }
#
# Error Handling:
# This function should throw an exception if the classify call fails for any reason
# or if the input csv file is of an improper format.
#
#TODO: Fill in this function
return
def compute_accuracy_of_single_classifier(classifier_dict, input_csv_file_name):
# Given a list of "classifications" for a given classifier, compute the accuracy of this
# classifier according to the input csv file
#
# Inputs:
# classifier_dict - A list of "classifications". Aka:
# A list of dictionaries, one for each text, in order of lines in the
# input file. Each element is a dictionary containing the top_class
# and the confidences of all the possible classes (ie the same
# format as returned by classify_single_text)
# Format example:
# [
# {'top_class': 'class_name',
# 'classes': [
# {'class_name': 'myclass', 'confidence': 0.999} ,
# {'class_name': 'myclass2', 'confidence': 0.001}
# ]
# },
# {'top_class': 'class_name',
# ...
# }
# ]
#
# input_csv_name - full path and name of an input csv file in the
# 6 column format of the input test/training files
#
# Returns:
# The accuracy of the classifier, as a fraction between [0.0-1.0] (ie percentage/100). \
# See the handout for more info.
#
# Error Handling:
# This function should throw an error if there is an issue with the
# inputs.
#
#TODO: fill in this function
return
def compute_average_confidence_of_single_classifier(classifier_dict, input_csv_file_name):
# Given a list of "classifications" for a given classifier, compute the average
# confidence of this classifier wrt the selected class, according to the input
# csv file.
#
# Inputs:
# classifier_dict - A list of "classifications". Aka:
# A list of dictionaries, one for each text, in order of lines in the
# input file. Each element is a dictionary containing the top_class
# and the confidences of all the possible classes (ie the same
# format as returned by classify_single_text)
# Format example:
# [
# {'top_class': 'class_name',
# 'classes': [
# {'class_name': 'myclass', 'confidence': 0.999} ,
# {'class_name': 'myclass2', 'confidence': 0.001}
# ]
# },
# {'top_class': 'class_name',
# ...
# }
# ]
#
# input_csv_name - full path and name of an input csv file in the
# 6 column format of the input test/training files
#
# Returns:
# The average confidence of the classifier, as a number between [0.0-1.0]
# See the handout for more info.
#
# Error Handling:
# This function should throw an error if there is an issue with the
# inputs.
#
#TODO: fill in this function
return
if __name__ == "__main__":
input_test_data = '<ADD FILE NAME HERE>'
#STEP 1: Ensure all 3 classifiers are ready for testing
#STEP 2: Test the test data on all classifiers
#STEP 3: Compute the accuracy for each classifier
#STEP 4: Compute the confidence of each class for each classifier