kendalls and spearmans evaluation for summe dataset implemented

src/evaluation/SumMeSummaryEvaluator.py

+import numpy as np
+from BaseEvaluator import BaseEvaluator
+
+
+class SumMeSummaryEvaluator(BaseEvaluator):
+    def __init__(self, generated_summaries, dataset, metric):
+        super().__init__(generated_summaries, dataset, metric)
+
+    def evaluate(self, metric='kendalltau'):
+        rc_func = self.get_rc_func(metric)
+        taus = []
+        for video_name in self.dataset.keys():
+            if video_name in self.summaries.keys():
+                tau, p_value = rc_func(self.dataset[video_name]['mean_user_score'], self.summaries[video_name])
+                taus.append(tau)
+        return np.asarray(taus).mean()

src/evaluation/evaluator.py

+import argparse
+import json
+from os import listdir
+
+import numpy as np
+from SumMeSummaryEvaluator import SumMeSummaryEvaluator
+from summary_loader import load_processed_summe
+
+PROCESSED_SUMME = '../../data/SumMe/processed/eccv16_dataset_summe_google_pool5.h5'
+PROCESSED_TVSUM = '../../data/TVSUM/processed/eccv16_dataset_tvsum_google_pool5.h5'
+
+
+def upsample_summary(summary_scores, n_frames, positions):
+    # compute the importance scores for the original length frame sequence
+    frame_scores = np.zeros((n_frames), dtype=np.float32)
+    if positions.dtype != int:
+        positions = positions.astype(np.int32)
+    if positions[-1] != n_frames:
+        positions = np.concatenate([positions, [n_frames]])
+
+    # fill the original length summary with the values of generated summary
+    for i in range(len(positions) - 1):
+        pos_left, pos_right = positions[i], positions[i + 1]
+        if i == len(summary_scores):
+            frame_scores[pos_left:pos_right] = 0
+        else:
+            frame_scores[pos_left:pos_right] = summary_scores[i]
+
+    return frame_scores
+
+
+def upsample_summaries(scores, processed_dataset):
+    upsampled_summaries = dict()
+    for video in scores.keys():
+        n_frames = np.array(processed_dataset[video]['nframes'])
+        positions = np.array(processed_dataset[video]['positions'])
+        video_scores = np.array(scores[video])
+        upsampled_summary = upsample_summary(video_scores, n_frames, positions)
+        upsampled_summaries[video] = upsampled_summary
+    return upsampled_summaries
+
+
+def load_results(path):
+    results = listdir(path)
+    results.sort(key=lambda video: int(video[6:-5]))
+    epoch0 = path + "/" + results[0]
+    video_names = resolve_video_names(epoch0)
+    all_scores = init_scores_dict(video_names)
+    for idx, epoch in enumerate(results):
+        with open(path + "/" + epoch) as f:
+            data = json.loads(f.read())
+            for video_name in video_names:
+                scores = data[video_name]
+                all_scores[video_name].append(scores)
+    return all_scores
+
+
+def average_results(video_scores):
+    mean_scores = dict()
+    for key in video_scores.keys():
+        mean_scores[key] = np.asarray(video_scores[key]).mean(axis=0)
+
+    return mean_scores
+
+
+def resolve_video_names(json_file):
+    with open(json_file) as f:
+        data = json.loads(f.read())
+        return list(data.keys())
+
+
+def init_scores_dict(keys):
+    all_scores = dict()
+    all_scores = {el: [] for el in keys}  # init dict
+    return all_scores
+
+
+def arg_parser():
+    parser = argparse.ArgumentParser(description="Generate Summary")
+    parser.add_argument("--results_path", type=str, default='../../results/SumMe/split0',
+                        help="Scores path from a split")
+    return parser
+
+
+if __name__ == "__main__":
+    parser = arg_parser()
+    args = parser.parse_args()
+    results_path = args.results_path
+    # load dataset
+    processed_summe = load_processed_summe()
+    # load results from split folder
+    video_scores = load_results(results_path)
+    # average the results over the total number of epochs
+    avg_res = average_results(video_scores)
+    # upsample the generated summary to equal the original
+    upsampled_summaries = upsample_summaries(avg_res, processed_summe)
+
+    metric = 'kendalltau'
+    summe_evaluator = SumMeSummaryEvaluator(upsampled_summaries, processed_summe, metric)
+    print(summe_evaluator.evaluate())
+

src/evaluation/summary_loader.py

+import h5py
+import hdf5storage
+import numpy as np
+
+SUMME_GT_DIR = '../../data/SumMe/GT'
+PROCESSED_SUMME = '../../data/SumMe/processed/eccv16_dataset_summe_google_pool5.h5'
+
+
+# PROCESSED_TVSUM = '../../data/TVSUM/processed/eccv16_dataset_tvsum_google_pool5.h5'
+
+
+def load_tvsum_mat(filename):
+    data = hdf5storage.loadmat(filename, variable_names=['tvsum50'])
+    data = data['tvsum50'].ravel()
+
+    data_list = []
+    for item in data:
+        video, category, title, length, nframes, user_anno, gt_score = item
+
+        item_dict = {
+            'video': video[0, 0],
+            'category': category[0, 0],
+            'title': title[0, 0],
+            'length': length[0, 0],
+            'nframes': nframes[0, 0],
+            'user_anno': user_anno,
+            'gt_score': gt_score
+        }
+
+        data_list.append(item_dict)
+
+    return data_list
+
+
+def load_processed_summe(processed_summe=PROCESSED_SUMME):
+    with h5py.File(processed_summe, 'r') as hdf:
+        data_list = dict()
+        for video_name in hdf.keys():
+            video_idx = video_name[6:]
+            elemnt = 'video_' + video_idx
+            nframes = np.array(hdf.get(elemnt + '/n_frames'))
+            positions = np.array(hdf.get(elemnt + '/picks'))
+            user_score = np.array(hdf.get(elemnt + '/user_summary'))
+            mean_user_score = np.asarray(user_score).mean(axis=0)
+            gt_score = np.array(hdf.get(elemnt + '/gtsummary'))
+            item_dict = {
+                'nframes': nframes,
+                'user_score': user_score,
+                'positions': positions,
+                'mean_user_score': mean_user_score,
+                'gt_score': gt_score,
+            }
+
+            data_list[elemnt] = item_dict
+
+    hdf.close()
+    return data_list