eval.py

import math
from statistics import mean

import sklearn
import sklearn.metrics
import torch

from process_data import _make_char_vector, _make_word_vector, gen_cmntrank_batches, gen_editanch_batches
from process_data import to_var, make_vector
from wiki_util import tokenizeText


## general evaluation
def predict(pred_cmnt, pred_ctx, w2i, c2i, model, max_ctx_length):
    print("prediction on single sample ...")
    model.eval()

    _, pred_cmnt_words = tokenizeText(pred_cmnt)
    pred_cmnt_chars = [list(i) for i in pred_cmnt_words]
    _, pred_ctx_words = tokenizeText(pred_ctx)
    pred_ctx_chars = [list(i) for i in pred_ctx_words]

    cmnt_sent_len = len(pred_cmnt_words)
    cmnt_word_len = int(mean([len(w) for w in pred_cmnt_chars]))
    ctx_sent_len = max_ctx_length
    ctx_word_len = int(mean([len(w) for w in pred_ctx_chars]))

    cmnt_words, cmnt_chars, ctx_words, ctx_chars = [], [], [], []
    # c, cc, q, cq, a in batch

    cmnt_words.append(_make_word_vector(pred_cmnt_words, w2i, cmnt_sent_len))
    cmnt_chars.append(_make_char_vector(pred_cmnt_chars, c2i, cmnt_sent_len, cmnt_word_len))
    ctx_words.append(_make_word_vector(pred_ctx_words, w2i, ctx_sent_len))
    ctx_chars.append(_make_char_vector(pred_ctx_chars, c2i, ctx_sent_len, ctx_word_len))

    cmnt_words = to_var(torch.LongTensor(cmnt_words))
    cmnt_chars = to_var(torch.stack(cmnt_chars, 0))
    ctx_words = to_var(torch.LongTensor(ctx_words))
    ctx_chars = to_var(torch.stack(ctx_chars, 0))

    logit, _ = model(ctx_words, ctx_chars, cmnt_words, cmnt_chars)
    a = torch.max(logit.cpu(), -1)
    print(logit)
    print(a)
    y_pred = a[1].data[0]
    y_prob = a[0].data[0]
    y_pred_2 = [int(i) for i in (torch.max(logit, -1)[1].view(1).data).tolist()][0]
    print(y_pred_2)
    # y_pred = y_pred[0]
    print(y_pred, y_prob)
    return y_pred


def compute_rank_score(pos_score, neg_scores):
    p1, p3, p5 = 0, 0, 0

    pos_list = [0 if pos_score > neg_score else 1 for neg_score in neg_scores]
    pos = sum(pos_list)
    # precision @K
    if pos == 0: p1 = 1
    if pos < 3: p3 = 1
    if pos < 5: p5 = 1
    # MRR
    mrr = 1 / (pos + 1)
    # NDCG: DCG/IDCG (In our case, we set the rel=1 if relevent, otherwise rel=0; Then IDCG=1)
    ndcg = 1 / math.log2(pos + 2)

    return p1, p3, p5, mrr, ndcg


def get_rank(pos_score, neg_scores):
    pos_list = [0 if pos_score > neg_score else 1 for neg_score in neg_scores]
    pos = sum(pos_list)
    return pos + 1


def isEditPredCorrect(pred, truth):
    for i in len(pred):
        if pred[i] != truth[i]:
            return False

    return True


def eval_rank(score_pos, score_neg, cand_num):
    score_pos_list = score_pos.data.cpu().squeeze(1).numpy().tolist()
    score_neg_list = score_neg.data.cpu().squeeze(1).numpy().tolist()

    correct_p1, correct_p3, correct_p5, total_mrr, total_ndcg = 0, 0, 0, 0, 0
    neg_num = cand_num - 1
    batch_num = int(len(score_neg) / neg_num)
    rank_list = []
    for i in range(batch_num):
        score_pos_i = score_pos_list[i * neg_num: (i + 1) * neg_num]
        score_neg_i = score_neg_list[i * neg_num: (i + 1) * neg_num]

        p1, p3, p5, mrr, ndcg = compute_rank_score(score_pos_i[0], score_neg_i)
        rank = get_rank(score_pos_i[0], score_neg_i)
        rank_list.append(rank)
        correct_p1 += p1
        correct_p3 += p3
        correct_p5 += p5
        total_mrr += mrr
        total_ndcg += ndcg

    return correct_p1, correct_p3, correct_p5, total_mrr, total_ndcg, rank_list


# def eval_rank_orig(score_pos, score_neg, batch_size):
#     score_pos_list = score_pos.data.cpu().squeeze(1).numpy().tolist()
#     score_neg_list = score_neg.data.cpu().squeeze(1).numpy().tolist()
#
#     total_p1, total_p3, total_p5, total_mrr, total_ndcg = 0, 0, 0, 0, 0
#     sample_num = int(len(score_neg) / batch_size)
#     for i in range(batch_size):
#         score_pos_i = score_pos_list[i * sample_num: (i+1) * sample_num]
#         score_neg_i = score_neg_list[i * sample_num: (i+1) * sample_num]
#         pos_list = [0 if score_pos_i[i] >= score_neg_i[i] else 1 for i in range(sample_num)]
#         pos = sum(pos_list)
#         sorted_neg = ["%.4f" % i for i in sorted(score_neg_i, reverse=True)]
#         #print(pos, "%.4f" % score_pos_i[0], "\t".join(sorted_neg), sep='\t')
#         if pos == 0:
#             total_p1 += 1
#
#         if pos < 3:
#             total_p3 += 1
#
#         if pos < 5:
#             total_p5 += 1
#
#         # MRR
#         total_mrr += 1 / (pos + 1)
#
#         # NDCG: DCG/IDCG (In our case, we set the rel=1 if relevent, otherwise rel=0; Then IDCG=1)
#         total_ndcg += 1 / math.log2(pos + 2)
#
#     return total_p1, total_p3, total_p5, total_mrr, total_ndcg

## general evaluation
def eval(dataset, val_df, w2i, model, args):
    # print(" evaluation on", val_df.shape[0], " samples ...")
    model.eval()
    corrects, avg_loss = 0, 0

    cmnt_rank_p1, cmnt_rank_p3, cmnt_rank_p5, cmnt_rank_mrr, cmnt_rank_ndcg = 0, 0, 0, 0, 0
    ea_pred, ea_truth = [], []
    cr_total, ea_total = 0, 0
    pred_cr_list, pred_ea_list = [], []
    for batch in dataset.iterate_minibatches(val_df, args.batch_size):
        cmnt_sent_len = args.max_cmnt_length
        ctx_sent_len = args.max_ctx_length
        diff_sent_len = args.max_diff_length

        ###########################################################
        # Comment Ranking Task
        ###########################################################
        # generate positive and negative batches
        pos_batch, neg_batch = gen_cmntrank_batches(batch, w2i, cmnt_sent_len, diff_sent_len, ctx_sent_len,
                                                    args.rank_num)

        pos_cmnt, pos_src_token, pos_src_action, pos_tgt_token, pos_tgt_action = \
            make_vector(pos_batch, w2i, cmnt_sent_len, ctx_sent_len)
        neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action = \
            make_vector(neg_batch, w2i, cmnt_sent_len, ctx_sent_len)
        score_pos, _ = model(pos_cmnt, pos_src_token, pos_src_action, pos_tgt_token, pos_tgt_action, cr_mode=True)
        score_neg, _ = model(neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action, cr_mode=True)

        cr_p1_corr, cr_p3_corr, cr_p5_corr, cr_mrr, cr_ndcg, pred_rank = eval_rank(score_pos, score_neg, args.rank_num)
        cmnt_rank_p1 += cr_p1_corr
        cmnt_rank_p3 += cr_p3_corr
        cmnt_rank_p5 += cr_p5_corr
        cmnt_rank_mrr += cr_mrr
        cmnt_rank_ndcg += cr_ndcg
        cr_total += int(len(score_pos) / (args.rank_num - 1))
        pred_cr_list += pred_rank

        ###########################################################
        # Edits Anchoring
        ###########################################################
        # generate positive and negative batches
        ea_batch, ea_truth_cur = gen_editanch_batches(batch, w2i, cmnt_sent_len, diff_sent_len, ctx_sent_len,
                                                      args.anchor_num)
        if len(pos_batch[0]) > 0:
            cmnt, src_token, src_action, tgt_token, tgt_action = \
                make_vector(ea_batch, w2i, cmnt_sent_len, ctx_sent_len)
            # neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action = \
            #                         make_vector(neg_batch, w2i, cmnt_sent_len, ctx_sent_len)
            logit, _ = model(cmnt, src_token, src_action, tgt_token, tgt_action, cr_mode=False)
            # logit_neg, _ = model(neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action, cr_mode=False)

            ea_pred_cur = (torch.max(logit, 1)[1].view(logit.size(0)).data).tolist()
            # ea_truth_cur = [1] * logit_pos.size(0) + [0] * logit_neg.size(0)

            ea_pred += ea_pred_cur
            ea_truth += ea_truth_cur

            ea_total += int(len(score_pos) / (args.anchor_num - 1))

    # # output the prediction results
    # with open(args.checkpoint_path + 'test_out.txt', 'w') as f:
    #     for i in range(len(y_truth)):
    #         line = cmnt_readable_all[i] + '\t' + ctx_readable_all[i] + '\t' + str(y_pred[i]) + '\t' + str(y_truth[i])
    #         f.write(line + '\n')

    # if args.test:
    #     print(total_rank)
    # print("\t".join([str(i) for i in pred_cr_list]))
    # print("\t".join([str(i) for i in ea_pred]))

    cr_p1_acc = cmnt_rank_p1 / cr_total
    cr_p3_acc = cmnt_rank_p3 / cr_total
    cr_p5_acc = cmnt_rank_p5 / cr_total
    cr_mrr = cmnt_rank_mrr / cr_total
    cr_ndcg = cmnt_rank_ndcg / cr_total

    ea_acc = (sklearn.metrics.accuracy_score(ea_truth, ea_pred))
    ea_f1 = (sklearn.metrics.f1_score(ea_truth, ea_pred, pos_label=1))
    ea_prec = (sklearn.metrics.precision_score(ea_truth, ea_pred, pos_label=1))
    ea_recall = (sklearn.metrics.recall_score(ea_truth, ea_pred, pos_label=1))

    print("\n*** Validation Results *** ")
    # print("[Task-CR] P@1:", "%.3f" % cr_p1_acc, "% P@3:", "%.3f" % cr_p3_acc, "% P@5:", "%.3f" % cr_p5_acc,\
    #         '%', ' (', cmnt_rank_p1, '/', cr_total, ',', cmnt_rank_p3, '/', cr_total, ',', cmnt_rank_p5, '/', cr_total,')', sep='')
    # print("[Task-RA] P@1:", "%.3f" % ea_p1_acc, "% P@3:", "%.3f" % ea_p3_acc, "% P@5:", "%.3f" % ea_p5_acc,\
    #         '%', ' (', edit_anch_p1, '/', ea_total, ',', edit_anch_p3, '/', ea_total, ',', edit_anch_p5, '/', ea_total,')', sep='')
    print("[Task-CR] P@1:", "%.3f" % cr_p1_acc, " P@3:", "%.3f" % cr_p3_acc, " P@5:", "%.3f" % cr_p5_acc, " MRR:",
          "%.3f" % cr_mrr, " NDCG:", "%.3f" % cr_ndcg, sep='')
    print("[Task-EA] ACC:", "%.3f" % ea_acc, " F1:", "%.3f" % ea_f1, " Precision:", "%.3f" % ea_prec, " Recall:",
          "%.3f" % ea_recall, sep='')
    return cr_p1_acc, ea_f1


def dump_cmntrank_case(pos_batch, neg_batch, idx, rank_num, diff_url, rank, pos_score, neg_scores):
    neg_num = rank_num - 1
    pos_cmnt = pos_batch[0][idx * neg_num]
    neg_cmnts = neg_batch[0][idx * neg_num: (idx + 1) * neg_num]

    before_edit = pos_batch[1][idx * neg_num]
    after_edit = pos_batch[3][idx * neg_num]

    match = False
    for token in pos_cmnt:
        if token in before_edit + after_edit:
            match = True
            break

    neg_match_words = []
    neg_match = False
    for neg_cmnt in neg_cmnts:
        for token in neg_cmnt:
            if len(token) <= 3:
                continue
            if token in before_edit + after_edit:
                neg_match = True
                neg_match_words.append(token)

    if not match and neg_match:
        print("\n ====== cmntrank case (Not Matched) ======")
        print("Rank", rank)
        print(diff_url)
        print("pos_cmnt (", "{0:.3f}".format(pos_score), "): ", " ".join(pos_cmnt), sep='')

        for i, neg_cmnt in enumerate(neg_cmnts):
            print("neg_cmnt ", i, " (", "{0:.3f}".format(neg_scores[i]), "): ", " ".join(neg_cmnt), sep='')
        pass
        print("neg_match_words:", " ".join(neg_match_words))


def dump_editanch_case(comment, edit, pred, truth):
    print("\n ====== editanch case ======")
    print("pred/truth: ", pred, "/", truth)
    print("comment:", " ".join(comment))
    print("edit:", " ".join(edit))


def case_study(dataset, val_df, w2i, model, args):
    model.eval()
    print("Start the case study")
    # for batch in dataset.iterate_minibatches(val_df[:500], args.batch_size):
    for batch in dataset.iterate_minibatches(val_df, args.batch_size):
        cmnt_sent_len = args.max_cmnt_length
        ctx_sent_len = args.max_ctx_length
        diff_sent_len = args.max_diff_length

        ###########################################################
        # Comment Ranking Task
        ###########################################################
        # generate positive and negative batches

        if args.cr_train:
            pos_batch, neg_batch = gen_cmntrank_batches(batch, w2i, cmnt_sent_len, diff_sent_len, ctx_sent_len,
                                                        args.rank_num)

            pos_cmnt, pos_src_token, pos_src_action, pos_tgt_token, pos_tgt_action = \
                make_vector(pos_batch, w2i, cmnt_sent_len, ctx_sent_len)
            neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action = \
                make_vector(neg_batch, w2i, cmnt_sent_len, ctx_sent_len)
            score_pos, _ = model(pos_cmnt, pos_src_token, pos_src_action, pos_tgt_token, pos_tgt_action, cr_mode=True)
            score_neg, _ = model(neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action, cr_mode=True)

            score_pos_list = score_pos.data.cpu().squeeze(1).numpy().tolist()
            score_neg_list = score_neg.data.cpu().squeeze(1).numpy().tolist()

            neg_num = args.rank_num - 1
            batch_num = int(len(score_neg) / neg_num)
            for i in range(batch_num):
                score_pos_i = score_pos_list[i * neg_num: (i + 1) * neg_num]
                score_neg_i = score_neg_list[i * neg_num: (i + 1) * neg_num]

                rank = get_rank(score_pos_i[0], score_neg_i)
                dump_cmntrank_case(pos_batch, neg_batch, i, args.rank_num, batch[8][i], rank, score_pos_i[0],
                                   score_neg_i)

        ###########################################################
        # Edits Anchoring
        ###########################################################
        # generate positive and negative batches

        if args.ea_train:
            ea_batch, ea_truth_cur = gen_editanch_batches(batch, w2i, cmnt_sent_len, diff_sent_len, ctx_sent_len,
                                                          args.anchor_num)
            cmnt, src_token, src_action, tgt_token, tgt_action = \
                make_vector(ea_batch, w2i, cmnt_sent_len, ctx_sent_len)
            # neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action = \
            #                         make_vector(neg_batch, w2i, cmnt_sent_len, ctx_sent_len)
            logit, _ = model(cmnt, src_token, src_action, tgt_token, tgt_action, cr_mode=False)
            # logit_neg, _ = model(neg_cmnt, neg_src_token, neg_src_action, neg_tgt_token, neg_tgt_action, cr_mode=False)
            ea_pred_cur = (torch.max(logit, 1)[1].view(logit.size(0)).data).tolist()

            for i in range(len(ea_truth_cur)):
                # if ea_pred_cur[i] == ea_truth_cur[i]:
                dump_editanch_case(ea_batch[0][i], ea_batch[3][i], ea_pred_cur[i], ea_truth_cur[i])

            pass
            # ea_truth_cur = [1] * logit_pos.size(0) + [0] * logit_neg.size(0)