-
Notifications
You must be signed in to change notification settings - Fork 0
/
benchmark.py
41 lines (33 loc) · 1.33 KB
/
benchmark.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
from paradox.metrics import pearson, mse
from paradox.pipeline import pipeline
from paradox import k_neighbors_regressor
import numpy as np
from paradox import similarity
from paradox import parser
def report(correlations, errors, y_pred_fold):
print("PC:\t\t\t%0.2f\t(+/- %0.2f)" % (np.mean(correlations),
np.std(correlations) * 2))
print("RMSE:\t\t\t%0.2f\t(+/- %0.2f)" % (np.mean(errors),
np.std(errors) * 2))
print('{}'.format('*' * 40))
def test(model, categories=[]):
print("Testing on Category {}".format(categories))
test_pairs = parser.parse(mode="test", categories=categories)
X = [pair[0] for pair in test_pairs]
y = [pair[1] for pair in test_pairs]
y_pred = model.predict(X)
pcs = [pearson(y, y_pred)]
rmses = [mse(y, y_pred)]
report(pcs, rmses, y_pred)
pairs = parser.parse(mode="train")
X = [pair[0] for pair in pairs]
y = [pair[1] for pair in pairs]
transformer = similarity.build(verbose=True)
estimator = k_neighbors_regressor.build(n_neighbors=4)
p = pipeline(transformers=[transformer], estimator=estimator)
p.fit(X, y)
test(p, categories=["answer-answer"])
test(p, categories=["question-question"])
test(p, categories=["headlines"])
test(p, categories=["postediting"])
test(p, categories=["plagiarism"])