t_stat.py

import pickle
import numpy as np
from scipy import stats
import argparse


def main(score_dict_a, score_dict_b, k_list, tag_list):
    for tag in tag_list:
        for k in k_list:
            f1_np_array_a = np.array(score_dict_a['f1_score@{}_{}'.format(k, tag)])
            f1_np_array_b = np.array(score_dict_b['f1_score@{}_{}'.format(k, tag)])
            t_stat, p_value = stats.ttest_rel(f1_np_array_a, f1_np_array_b)
            print("tag: {}, topk: {}, p-value: {}".format(tag, k, p_value))


if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='t_stat')
    parser.add_argument('-score_dict_path_a', type=str, default='')
    parser.add_argument('-score_dict_path_b', type=str, default='')
    parser.add_argument('-k_list', nargs='+', default=['5', 'M'], type=str, help='only allow integer or M')
    opt = parser.parse_args()
    pickle_in_a = open(opt.score_dict_path_a, "rb")
    pickle_in_b = open(opt.score_dict_path_b, "rb")
    score_dict_a = pickle.load(pickle_in_a)
    score_dict_b = pickle.load(pickle_in_b)
    tag_list = ['present', 'absent']
    main(score_dict_a, score_dict_b, opt.k_list, tag_list)