测试map
上一篇训练教程mmrotate框架训练数据集，选用的iou=50，
我想测试iou在75之下的map的值，和iou在0.05-0.95递增的情况的平均map，需要进行以下操作：
测试AP75
1.修改mmrotate/datasets/dota.py
主要有一个参数：
1）iou_thr: iou阈值，修改成75
2.修改 test.py 中的参数
主要有三个参数：
1）config: 使用的模型文件 ；
2）checkpoint：训练得到的模型权重文件;
3）show-dir: 预测结果存放的路径
补充：测试用的数据集就是训练的步骤4中的数据集路径
3.输入指令
python test.py --eval mAP测试AP（iou在0.05-0.95递增的情况的平均map）
需要借助其他的代码
1.修改mmrotate/datasets/dota.py
在类class DOTADataset(CustomDataset)下添加以下子函数
def save_dota_txt(self, results, outfolder,conf_thresh=0.01): import math if not os.path.exists(outfolder): os.mkdir(outfolder) for i in range(len(self)): imgname = self.data_infos[i]['filename'] bboxresult = results[i] txt_name = os.path.join(outfolder,imgname.split('.')[0]+'.txt') f = open(txt_name, 'w') for cls, bboxes in zip(self.CLASSES, bboxresult): for box in bboxes: if box[-1] < conf_thresh: continue xc, yc, w, h, ag, score = box.tolist() wx, wy = w / 2 * math.cos(ag), w / 2 * math.sin(ag) hx, hy = -h / 2 * math.sin(ag), h / 2 * math.cos(ag) p1 = (xc - wx - hx, yc - wy - hy) p2 = (xc + wx - hx, yc + wy - hy) p3 = (xc + wx + hx, yc + wy + hy) p4 = (xc - wx + hx, yc - wy + hy) ps = (np.concatenate([p1, p2, p3, p4])) points = ' '.join(ps.astype(float).astype(str)) # soft label and eval result f.write(points + ' ' + cls + ' ' + str(box[-1]) + '\n') # hard label # if float(box[-1]) > 0.1: # f.write(points + ' ' + cls + ' 0\n') f.close()2.借用evaluation.py，修改map步长
import numpy as npimport osfrom shapely.geometry import Polygonimport syssys.path.append('/home/disk/bing/mmrotate-main/DOTA_devkit')import polyioudef parse_gt(filename): """ :param filename: ground truth file to parse :return: all instances in a picture """ objects = [] with open(filename, 'r') as f: while True: line = f.readline() if line: splitlines = line.strip().split(' ') object_struct = {} if (len(splitlines) < 9): continue object_struct['name'] = splitlines[8] object_struct['difficult'] = float(splitlines[9]) object_struct['bbox'] = [ float(splitlines[0]), float(splitlines[1]), float(splitlines[2]), float(splitlines[3]), float(splitlines[4]), float(splitlines[5]), float(splitlines[6]), float(splitlines[7]) ] objects.append(object_struct) else: break return objectsdef voc_ap(rec, prec, use_07_metric=False): """ ap = voc_ap(rec, prec, [use_07_metric]) Compute VOC AP given precision and recall. If use_07_metric is true, uses the VOC 07 11 point method (default:False). """ if use_07_metric: # 11 point metric ap = 0. for t in np.arange(0., 1.1, 0.1): if np.sum(rec >= t) == 0: p = 0 else: p = np.max(prec[rec >= t]) ap = ap + p / 11. else: # correct AP calculation # first append sentinel values at the end mrec = np.concatenate(([0.], rec, [1.])) mpre = np.concatenate(([0.], prec, [0.])) # compute the precision envelope for i in range(mpre.size - 1, 0, -1): mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i]) # to calculate area under PR curve, look for points # where X axis (recall) changes value i = np.where(mrec[1:] != mrec[:-1])[0] # and sum (\Delta recall) * prec ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1]) return apdef parse_predict(detpath, classname): image_ids = [] confidence = [] BB = [] files = os.listdir(detpath) for file in files: image_name = file.split('.')[0] R = [ obj for obj in parse_gt(os.path.join(detpath, file)) if obj['name'] == classname ] bbox = [x['bbox'] for x in R] scores = [x['difficult'] for x in R] image_names = [image_name for x in R] image_ids += image_names confidence += scores BB += bbox return np.array(image_ids), np.array(confidence), np.array(BB)def cal_iou(g, p): g = np.asarray(g) p = np.asarray(p) g = Polygon(g[:8].reshape((4, 2))) #print('p:', p) p = Polygon(p[:8].reshape((4, 2))) if not g.is_valid or not p.is_valid: return 0 inter = Polygon(g).intersection(Polygon(p)).area union = g.area + p.area - inter if union == 0: return 0 else: return inter / uniondef voc_eval( detpath, annopath, classname, # cachedir, ovthresh=0.5, use_07_metric=False): recs = {} for txt in os.listdir(annopath): recs[txt[:-4]] = parse_gt(os.path.join(annopath, txt)) class_recs = {} npos = 0 for txt in os.listdir(annopath): R = [obj for obj in recs[txt[:-4]] if obj['name'] == classname] bbox = np.array([x['bbox'] for x in R]) difficult = np.array([False for x in R]) det = [False] * len(R) npos = npos + len(R) class_recs[txt[:-4]] = { 'bbox': bbox, 'difficult': difficult, 'det': det } image_ids, confidence, BB = parse_predict(detpath, classname) sorted_ind = np.argsort(-confidence) if BB.shape[0] == 0: return 0, 0, 0 BB = BB[sorted_ind, :] image_ids = [image_ids[x] for x in sorted_ind] nd = len(image_ids) tp = np.zeros(nd) fp = np.zeros(nd) for d in range(nd): R = class_recs[image_ids[d]] bb = BB[d, :].astype(float) ovmax = -np.inf BBGT = R['bbox'].astype(float) ## compute det bb with each BBGT if BBGT.size > 0: # overlaps = np.array([cal_iou(bb_GT, bb) for bb_GT in BBGT]) # ovmax = np.max(overlaps) # jmax = np.argmax(overlaps) BBGT_xmin = np.min(BBGT[:, 0::2], axis=1) BBGT_ymin = np.min(BBGT[:, 1::2], axis=1) BBGT_xmax = np.max(BBGT[:, 0::2], axis=1) BBGT_ymax = np.max(BBGT[:, 1::2], axis=1) bb_xmin = np.min(bb[0::2]) bb_ymin = np.min(bb[1::2]) bb_xmax = np.max(bb[0::2]) bb_ymax = np.max(bb[1::2]) ixmin = np.maximum(BBGT_xmin, bb_xmin) iymin = np.maximum(BBGT_ymin, bb_ymin) ixmax = np.minimum(BBGT_xmax, bb_xmax) iymax = np.minimum(BBGT_ymax, bb_ymax) iw = np.maximum(ixmax - ixmin + 1., 0.) ih = np.maximum(iymax - iymin + 1., 0.) inters = iw * ih # union uni = ((bb_xmax - bb_xmin + 1.) * (bb_ymax - bb_ymin + 1.) + (BBGT_xmax - BBGT_xmin + 1.) * (BBGT_ymax - BBGT_ymin + 1.) - inters) overlaps = inters / uni BBGT_keep_mask = overlaps > 0 BBGT_keep = BBGT[BBGT_keep_mask, :] BBGT_keep_index = np.where(overlaps > 0)[0] # pdb.set_trace() def calcoverlaps(BBGT_keep, bb): overlaps = [] for index, GT in enumerate(BBGT_keep): overlap = polyiou.iou_poly( polyiou.VectorDouble(BBGT_keep[index]), polyiou.VectorDouble(bb)) overlaps.append(overlap) return overlaps if len(BBGT_keep) > 0: overlaps = calcoverlaps(BBGT_keep, bb) ovmax = np.max(overlaps) jmax = np.argmax(overlaps) # pdb.set_trace() jmax = BBGT_keep_index[jmax] if ovmax > ovthresh: if not R['difficult'][jmax]: if not R['det'][jmax]: tp[d] = 1. R['det'][jmax] = 1 else: fp[d] = 1. else: fp[d] = 1. # print('check fp:', sum(fp)) # print('check tp', sum(tp)) # print('npos num:', npos) fp = np.cumsum(fp) tp = np.cumsum(tp) rec = tp / float(npos) prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps) ap = voc_ap(rec, prec, use_07_metric) return rec, prec, apdef evalution(detpath,annopath): # predict folder # detpath = '/home/cv123/data/zyj/ReDet_back/eval/predota_kaggle768_on_fair' print(detpath) print(annopath) # # gt folder # classes = ['plane', 'baseball-diamond', 'bridge', 'ground-track-field', # 'small-vehicle', 'large-vehicle', 'ship', 'tennis-court', # 'basketball-court', 'storage-tank', 'soccer-ball-field', # 'roundabout', 'harbor', 'swimming-pool', 'helicopter'] classes = ['ship',] ap_50 = 0 ap_75 = 0 map = 0 for cls in classes: aps = [] for iou_thr in [0.5, 0.55, 0.6, 0.65, 0.7, 0.75, 0.8, 0.85, 0.9, 0.95]: rec, prec, ap = voc_eval( detpath, annopath, cls, ovthresh=iou_thr, use_07_metric=True) aps.append(ap * 100) ap_50 += aps[0] ap_75 += aps[5] map += np.array(aps).mean() print(cls, ' ', 'AP50: {:.2f}\tAP75: {:.2f}'.format(aps[0], aps[5])) print('AP50: {:.2f}\tAP75: {:.2f}\t mAP: {:.2f}'.format( ap_50 / len(classes), ap_75 / len(classes), map / len(classes)))主要有两个参数：
1）sys.path.append(‘/home/disk/bing/mmrotate-main/DOTA_devkit’)
需要借用DOTA_devkit，一般将evaluation.py放在DOTA_devkit文件夹的同级目录下
2）classes = [‘ship’,]
3.修改DOTA_devkit/ResultMerge_multi_process.py
修改sys.path.append(‘/home/disk/bing/mmrotate-main/DOTA_devkit’)路径
4.借用test.py
import argparseimport osimport os.path as ospimport timeimport warningsimport mmcvimport torchfrom mmcv import Config, DictActionfrom mmcv.cnn import fuse_conv_bnfrom mmcv.parallel import MMDataParallel, MMDistributedDataParallelfrom mmcv.runner import (get_dist_info, init_dist, load_checkpoint, wrap_fp16_model)from mmdet.apis import multi_gpu_test, single_gpu_testfrom mmdet.datasets import build_dataloader, replace_ImageToTensorfrom mmrotate.datasets import build_datasetfrom mmrotate.models import build_detectorfrom mmrotate.utils import setup_multi_processes, compat_cfgfrom DOTA_devkit.ResultMerge_multi_process import mergebypoly2from evaluation import evalutiondef forward(config_path,checkpointpath,test_img_path,outpath,imgsize,batchsize): cfg = Config.fromfile(config_path) cfg = compat_cfg(cfg) # set multi-process settings setup_multi_processes(cfg) cfg.model.pretrained = None if cfg.model.get('neck'): if isinstance(cfg.model.neck, list): for neck_cfg in cfg.model.neck: if neck_cfg.get('rfp_backbone'): if neck_cfg.rfp_backbone.get('pretrained'): neck_cfg.rfp_backbone.pretrained = None elif cfg.model.neck.get('rfp_backbone'): if cfg.model.neck.rfp_backbone.get('pretrained'): cfg.model.neck.rfp_backbone.pretrained = None samples_per_gpu = batchsize distributed = False test_dict=dict( type='DOTADataset', test_mode = True, img_prefix=test_img_path, ann_file=test_img_path, pipeline=[ dict(type='LoadImageFromFile'), dict( type='MultiScaleFlipAug', img_scale=(imgsize, imgsize), flip=False, transforms=[ dict(type='RResize'), dict( type='Normalize', mean=[123.675, 116.28, 103.53], std=[58.395, 57.12, 57.375], to_rgb=True), dict(type='Pad', size_divisor=32), dict(type='DefaultFormatBundle'), dict(type='Collect', keys=['img']) ]) ], version='le90') dataset = build_dataset(test_dict) data_loader = build_dataloader( dataset, samples_per_gpu=samples_per_gpu, workers_per_gpu=min(20, samples_per_gpu), dist=distributed, shuffle=False) # build the model and load checkpoint cfg.model.train_cfg = None model = build_detector(cfg.model, test_cfg=cfg.get('test_cfg')) fp16_cfg = cfg.get('fp16', None) if fp16_cfg is not None: wrap_fp16_model(model) checkpoint = load_checkpoint(model, checkpointpath, map_location='cpu') if 'CLASSES' in checkpoint.get('meta', {}): model.CLASSES = checkpoint['meta']['CLASSES'] else: model.CLASSES = dataset.CLASSES if not distributed: model = MMDataParallel(model, device_ids=range(1)) outputs = single_gpu_test(model, data_loader) dataset.save_dota_txt(outputs,outfolder=outpath)if __name__ == '__main__': #cfg and checkpoint config_path = "/home/disk/bing/mmrotate-main/configs/oriented_reppoints/oriented_reppoints_r50_fpn_1x_dota_le135.py" checkpointpath = "/home/disk/bing/mmrotate-main/run/oriented_reppoints/epoch_12.pth" #test annopath = "/home/disk/bing/datasets/dota/val/labels" test_img_path = "/home/disk/bing/datasets/dota/val/images" #output txt folder outpath = "/home/disk/bing/datasets/dota/val/oriented_reppoints_epoch12_test" img_size = 1024 batchsize = 2 #forward forward(config_path,checkpointpath,test_img_path,outpath,img_size,batchsize) dst_folder = outpath evalution(dst_folder, annopath)主要有五个参数：
1）config_path
2）checkpointpath
3）annopath
4）test_img_path
5）outpath
一般将test.py放在evaluation.py的同级目录下
5.输入指令
python test.py