Initial commit

This commit is contained in:
2026-03-31 13:28:42 +02:00
commit 94bbdca745
224 changed files with 83161 additions and 0 deletions

View File

@@ -0,0 +1,71 @@
# Datasets
59.26TB of research data: http://academictorrents.com/
ImageNet Torrent (Stanford): http://academictorrents.com/browse.php?search=imagenet&page=0
25 thousand datasets on Kaggle: https://www.kaggle.com/datasets
BDD100K - Diverse Driving Video (Berkeley): https://bair.berkeley.edu/blog/2018/05/30/bdd/
KITTI - for autonomous driving (Toyota): http://www.cvlibs.net/datasets/kitti/
A2D2 - for autonomous driving (Audi): https://www.a2d2.audi/a2d2/en.html
nuScenes (for autonomous driving): https://www.nuscenes.org/overview
Pascal VOC (Oxford): http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html
MS COCO (Microsoft): http://cocodataset.org/#download
ImageNet (Stanford): http://imagenet.stanford.edu/download.php
ImageNet (ILSVRC2012): http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads
ImageNet (ILSVRC2015): http://image-net.org/small/download.php
ImageNet VID: http://bvisionweb1.cs.unc.edu/ilsvrc2015/download-videos-3j16.php
Open Images (Google): https://storage.googleapis.com/openimages/web/download.html
Cityscapes: https://www.cityscapes-dataset.com/
Object Tracking Benchmark: http://cvlab.hanyang.ac.kr/tracker_benchmark/datasets.html
MOT (Multiple object tracking benchmark): https://motchallenge.net/
VOT (Visual object tracking): http://www.votchallenge.net/challenges.html
FREE FLIR Thermal Dataset (infrared): https://www.flir.eu/oem/adas/adas-dataset-form/
MARS: http://www.liangzheng.com.cn/Project/project_mars.html
Market-1501: http://www.liangzheng.org/Project/project_reid.html
German Traffic Sign Recognition Benchmark: http://benchmark.ini.rub.de/
Labeled Faces in the Wild: http://vis-www.cs.umass.edu/lfw/
Core50: https://vlomonaco.github.io/core50/
Visual Question Answering: https://visualqa.org/download.html
Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/
----
Wikipedia's List of datasets: https://en.wikipedia.org/wiki/List_of_datasets_for_machine-learning_research
Other datasets (Music, Natural Images, Artificial Datasets, Faces, Text, Speech, Recommendation Systems, Misc): http://deeplearning.net/datasets/
25 datasets: https://www.analyticsvidhya.com/blog/2018/03/comprehensive-collection-deep-learning-datasets/
List of datasets: https://riemenschneider.hayko.at/vision/dataset/index.php
Another list of datasets: http://homepages.inf.ed.ac.uk/rbf/CVonline/Imagedbase.htm
Pedestrian DATASETs for Vision based Detection and Tracking: https://hemprasad.wordpress.com/2014/11/08/pedestrian-datasets-for-vision-based-detection-and-tracking/
TrackingNet: https://tracking-net.org/
RGB, RGBD, Texture-mapped 3D mesh models: http://www.ycbbenchmarks.com/

View File

@@ -0,0 +1,78 @@
#!/usr/bin/env pwsh
param (
[switch]$DisableVisualStudioFeatures = $false,
[switch]$DisableSilentMode = $false
)
Import-Module -Name $PSScriptRoot/utils.psm1 -Force
$url = "https://developer.download.nvidia.com/compute/cuda/${cuda_version_full}/network_installers/cuda_${cuda_version_full}_windows_network.exe"
$CudaFeatures = " nvcc_${cuda_version_short} " + `
" cublas_${cuda_version_short} " + `
" cublas_dev_${cuda_version_short} " + `
" cuda_profiler_api_${cuda_version_short} " + `
" cudart_${cuda_version_short} " + `
" cufft_${cuda_version_short} " + `
" cufft_dev_${cuda_version_short} " + `
" cuobjdump_${cuda_version_short} " + `
" cupti_${cuda_version_short} " + `
" curand_${cuda_version_short} " + `
" curand_dev_${cuda_version_short} " + `
" cusolver_${cuda_version_short} " + `
" cusolver_dev_${cuda_version_short} " + `
" cusparse_${cuda_version_short} " + `
" cusparse_dev_${cuda_version_short} " + `
" cuxxfilt_${cuda_version_short} " + `
" npp_${cuda_version_short} " + `
" npp_dev_${cuda_version_short} " + `
" nsight_compute_${cuda_version_short} " + `
" nsight_systems_${cuda_version_short} " + `
" nsight_vse_${cuda_version_short} " + `
" nvdisasm_${cuda_version_short} " + `
" nvjitlink_${cuda_version_short} " + `
" nvjpeg_${cuda_version_short} " + `
" nvjpeg_dev_${cuda_version_short} " + `
" nvml_dev_${cuda_version_short} " + `
" nvprof_${cuda_version_short} " + `
" nvprune_${cuda_version_short} " + `
" nvrtc_${cuda_version_short} " + `
" nvrtc_dev_${cuda_version_short} " + `
" nvtx_${cuda_version_short} " + `
" occupancy_calculator_${cuda_version_short} " + `
" opencl_${cuda_version_short} " + `
" sanitizer_${cuda_version_short} " + `
" thrust_${cuda_version_short} " + `
" visual_profiler_${cuda_version_short} "
if (-Not $DisableVisualStudioFeatures) {
$CudaFeatures = $CudaFeatures + "visual_studio_integration_${cuda_version_short} visual_profiler_${cuda_version_short} "
}
if ($DisableSilentMode) {
$SilentFlag = ' '
}
else {
$SilentFlag = '-s '
}
try {
Push-Location $PSScriptRoot
Write-Host "Downloading CUDA from $url..."
curl.exe -L -o cuda_${cuda_version_full}_windows_network.exe -s -S $url
Write-Host 'Installing CUDA...'
$proc = Start-Process -PassThru -FilePath "./cuda_${cuda_version_full}_windows_network.exe" -ArgumentList @($SilentFlag + $CudaFeatures)
$proc.WaitForExit()
$exitCode = $proc.ExitCode
Pop-Location
if ($exitCode -eq 0) {
Write-Host 'Installation successful!'
}
else {
Throw "Installation failed! Exited with $exitCode."
}
}
catch {
Throw "Failed to install CUDA! $($_.Exception.Message)"
}

View File

@@ -0,0 +1,37 @@
#!/usr/bin/env bash
if [[ "$OSTYPE" == "darwin"* ]]; then
echo "Unable to deploy CUDA on macOS, please wait for a future script update"
exit 1
elif [[ $(cut -f2 <<< $(lsb_release -i)) == "Ubuntu" ]]; then
distr_name="$(cut -f2 <<< $(lsb_release -i) | tr '[:upper:]' '[:lower:]')$(cut -f2 <<< $(lsb_release -r) | tr -d '.')"
else
echo "Unable to deploy CUDA on this OS, please wait for a future script update"
exit 2
fi
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
if [ -f $script_dir/requested_cuda_version.sh ]; then
source $script_dir/requested_cuda_version.sh
else
echo "Unable to find requested_cuda_version.sh script"
exit 3
fi
sudo apt-key del 7fa2af80
wget https://developer.download.nvidia.com/compute/cuda/repos/$distr_name/x86_64/cuda-keyring_1.1-1_all.deb
sudo dpkg -i cuda-keyring_1.1-1_all.deb
sudo apt-get update
sudo apt-get install -y --no-install-recommends build-essential g++
sudo apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg software-properties-common wget
sudo apt-get install -y --no-install-recommends zlib1g
sudo apt-get dist-upgrade -y
sudo apt-get install -y --no-install-recommends cuda-${CUDA_VERSION_DASHED}
sudo apt-get install -y --no-install-recommends libcudnn8
sudo apt-get install -y --no-install-recommends libcudnn8-dev
sudo rm -rf /usr/local/cuda
sudo ln -s /usr/local/cuda-${CUDA_VERSION} /usr/local/cuda
sudo apt-get clean

View File

@@ -0,0 +1,20 @@
mkdir -p images
mkdir -p images/orig
mkdir -p images/train
mkdir -p images/val
ffmpeg -i Face1.mp4 images/orig/face1_%6d.jpg
ffmpeg -i Face2.mp4 images/orig/face2_%6d.jpg
ffmpeg -i Face3.mp4 images/orig/face3_%6d.jpg
ffmpeg -i Face4.mp4 images/orig/face4_%6d.jpg
ffmpeg -i Face5.mp4 images/orig/face5_%6d.jpg
ffmpeg -i Face6.mp4 images/orig/face6_%6d.jpg
mogrify -resize 100x100^ -gravity center -crop 100x100+0+0 +repage images/orig/*
ls images/orig/* | shuf | head -n 1000 | xargs mv -t images/val
mv images/orig/* images/train
find `pwd`/images/train > dice.train.list -name \*.jpg
find `pwd`/images/val > dice.val.list -name \*.jpg

View File

@@ -0,0 +1,37 @@
#!/usr/bin/env pwsh
$url = "https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v4_pre/yolov4-tiny.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov4-tiny.weights"
$url = "https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov4.weights"
$url = "https://drive.google.com/u/0/uc?id=18yYZWyKbo4XSDVyztmsEcF9B_6bxrhUY&export=download"
Invoke-WebRequest -Uri $url -OutFile "yolov3-tiny-prn.weights"
$url = "https://pjreddie.com/media/files/yolov3.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov3.weights"
$url = "https://pjreddie.com/media/files/yolov3-openimages.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov3-openimages.weights"
$url = "https://pjreddie.com/media/files/yolov2.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov2.weights"
$url = "https://pjreddie.com/media/files/yolov3-tiny.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov3-tiny.weights"
$url = "https://pjreddie.com/media/files/yolov2-tiny.weights"
Invoke-WebRequest -Uri $url -OutFile "yolov2-tiny.weights"
$url = "https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v4_pre/yolov4-tiny.conv.29"
Invoke-WebRequest -Uri $url -OutFile "yolov4-tiny.conv.29"
$url = "https://github.com/AlexeyAB/darknet/releases/download/darknet_yolo_v3_optimal/yolov4.conv.137"
Invoke-WebRequest -Uri $url -OutFile "yolov4.conv.137"
$url = "https://pjreddie.com/media/files/darknet53.conv.74"
Invoke-WebRequest -Uri $url -OutFile "darknet53.conv.74"
$url = "https://pjreddie.com/media/files/darknet19_448.conv.23"
Invoke-WebRequest -Uri $url -OutFile "darknet19_448.conv.23"

View File

@@ -0,0 +1,165 @@
'''
Created on Feb 20, 2017
@author: jumabek
'''
from os import listdir
from os.path import isfile, join
import argparse
#import cv2
import numpy as np
import sys
import os
import shutil
import random
import math
width_in_cfg_file = 416.
height_in_cfg_file = 416.
def IOU(x,centroids):
similarities = []
k = len(centroids)
for centroid in centroids:
c_w,c_h = centroid
w,h = x
if c_w>=w and c_h>=h:
similarity = w*h/(c_w*c_h)
elif c_w>=w and c_h<=h:
similarity = w*c_h/(w*h + (c_w-w)*c_h)
elif c_w<=w and c_h>=h:
similarity = c_w*h/(w*h + c_w*(c_h-h))
else: #means both w,h are bigger than c_w and c_h respectively
similarity = (c_w*c_h)/(w*h)
similarities.append(similarity) # will become (k,) shape
return np.array(similarities)
def avg_IOU(X,centroids):
n,d = X.shape
sum = 0.
for i in range(X.shape[0]):
#note IOU() will return array which contains IoU for each centroid and X[i] // slightly ineffective, but I am too lazy
sum+= max(IOU(X[i],centroids))
return sum/n
def write_anchors_to_file(centroids,X,anchor_file):
f = open(anchor_file,'w')
anchors = centroids.copy()
print(anchors.shape)
for i in range(anchors.shape[0]):
anchors[i][0]*=width_in_cfg_file/32.
anchors[i][1]*=height_in_cfg_file/32.
widths = anchors[:,0]
sorted_indices = np.argsort(widths)
print('Anchors = ', anchors[sorted_indices])
for i in sorted_indices[:-1]:
f.write('%0.2f,%0.2f, '%(anchors[i,0],anchors[i,1]))
#there should not be comma after last anchor, that's why
f.write('%0.2f,%0.2f\n'%(anchors[sorted_indices[-1:],0],anchors[sorted_indices[-1:],1]))
f.write('%f\n'%(avg_IOU(X,centroids)))
print()
def kmeans(X,centroids,eps,anchor_file):
N = X.shape[0]
iterations = 0
k,dim = centroids.shape
prev_assignments = np.ones(N)*(-1)
iter = 0
old_D = np.zeros((N,k))
while True:
D = []
iter+=1
for i in range(N):
d = 1 - IOU(X[i],centroids)
D.append(d)
D = np.array(D) # D.shape = (N,k)
print("iter {}: dists = {}".format(iter,np.sum(np.abs(old_D-D))))
#assign samples to centroids
assignments = np.argmin(D,axis=1)
if (assignments == prev_assignments).all() :
print("Centroids = ",centroids)
write_anchors_to_file(centroids,X,anchor_file)
return
#calculate new centroids
centroid_sums=np.zeros((k,dim),np.float)
for i in range(N):
centroid_sums[assignments[i]]+=X[i]
for j in range(k):
centroids[j] = centroid_sums[j]/(np.sum(assignments==j))
prev_assignments = assignments.copy()
old_D = D.copy()
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument('-filelist', default = '\\path\\to\\voc\\filelist\\train.txt',
help='path to filelist\n' )
parser.add_argument('-output_dir', default = 'generated_anchors/anchors', type = str,
help='Output anchor directory\n' )
parser.add_argument('-num_clusters', default = 0, type = int,
help='number of clusters\n' )
args = parser.parse_args()
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
f = open(args.filelist)
lines = [line.rstrip('\n') for line in f.readlines()]
annotation_dims = []
size = np.zeros((1,1,3))
for line in lines:
#line = line.replace('images','labels')
#line = line.replace('img1','labels')
line = line.replace('JPEGImages','labels')
line = line.replace('.jpg','.txt')
line = line.replace('.png','.txt')
print(line)
f2 = open(line)
for line in f2.readlines():
line = line.rstrip('\n')
w,h = line.split(' ')[3:]
#print(w,h)
annotation_dims.append(tuple(map(float,(w,h))))
annotation_dims = np.array(annotation_dims)
eps = 0.005
if args.num_clusters == 0:
for num_clusters in range(1,11): #we make 1 through 10 clusters
anchor_file = join( args.output_dir,'anchors%d.txt'%(num_clusters))
indices = [ random.randrange(annotation_dims.shape[0]) for i in range(num_clusters)]
centroids = annotation_dims[indices]
kmeans(annotation_dims,centroids,eps,anchor_file)
print('centroids.shape', centroids.shape)
else:
anchor_file = join( args.output_dir,'anchors%d.txt'%(args.num_clusters))
indices = [ random.randrange(annotation_dims.shape[0]) for i in range(args.num_clusters)]
centroids = annotation_dims[indices]
kmeans(annotation_dims,centroids,eps,anchor_file)
print('centroids.shape', centroids.shape)
if __name__=="__main__":
main(sys.argv)

View File

@@ -0,0 +1,5 @@
#!/bin/bash
# Usage:
# wget http://pjreddie.com/media/files/peek.weights
# scripts/gen_tactic.sh < data/goal.txt
./darknet rnn generatetactic cfg/gru.cfg peek.weights 2>/dev/null

View File

@@ -0,0 +1,25 @@
#!/bin/bash
# Zip coco folder
# zip -r coco.zip coco
# tar -czvf coco.tar.gz coco
# Download labels from Google Drive, accepting presented query
filename="coco2017labels.zip"
fileid="1cXZR_ckHki6nddOmcysCuuJFM--T-Q6L"
curl -c ./cookie -s -L "https://drive.google.com/uc?export=download&id=${fileid}" > /dev/null
curl -Lb ./cookie "https://drive.google.com/uc?export=download&confirm=`awk '/download/ {print $NF}' ./cookie`&id=${fileid}" -o ${filename}
rm ./cookie
# Unzip labels
unzip -q ${filename} # for coco.zip
# tar -xzf ${filename} # for coco.tar.gz
rm ${filename}
# Download and unzip images
cd coco/images
f="train2017.zip" && curl http://images.cocodataset.org/zips/$f -o $f && unzip -q $f && rm $f # 19G, 118k images
f="val2017.zip" && curl http://images.cocodataset.org/zips/$f -o $f && unzip -q $f && rm $f # 1G, 5k images
# f="test2017.zip" && curl http://images.cocodataset.org/zips/$f -o $f && unzip -q $f && rm $f # 7G, 41k images
# cd out
cd ../..

View File

@@ -0,0 +1,34 @@
#!/bin/bash
# Clone COCO API
git clone https://github.com/pdollar/coco
cd coco
mkdir images
cd images
# Download Images
#very slow downloading
#wget -c https://pjreddie.com/media/files/train2014.zip
#wget -c https://pjreddie.com/media/files/val2014.zip
wget -c http://images.cocodataset.org/zips/train2014.zip
wget -c http://images.cocodataset.org/zips/val2014.zip
# Unzip
unzip -q train2014.zip
unzip -q val2014.zip
cd ..
# Download COCO Metadata
wget -c https://pjreddie.com/media/files/instances_train-val2014.zip
wget -c https://pjreddie.com/media/files/coco/5k.part
wget -c https://pjreddie.com/media/files/coco/trainvalno5k.part
wget -c https://pjreddie.com/media/files/coco/labels.tgz
tar xzf labels.tgz
unzip -q instances_train-val2014.zip
# Set Up Image Lists
paste <(awk "{print \"$PWD\"}" <5k.part) 5k.part | tr -d '\t' > 5k.txt
paste <(awk "{print \"$PWD\"}" <trainvalno5k.part) trainvalno5k.part | tr -d '\t' > trainvalno5k.txt

View File

@@ -0,0 +1,17 @@
#!/bin/bash
wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar
mkdir -p ILSVRC2012_img_train
tar --force-local -xf ILSVRC2012_img_train.tar -C ILSVRC2012_img_train
wd=`pwd`
for f in ILSVRC2012_img_train/*.tar;
do
name=$(echo "$f" | cut -f 1 -d '.')
mkdir "${wd}/${name}"
tar --force-local -xf "${wd}/${f}" -C "${wd}/${name}"
done
find "${wd}/ILSVRC2012_img_train" -name \*.JPEG > imagenet1k.train.list

View File

@@ -0,0 +1,19 @@
import csv
import os
#select classes you want to download at https://github.com/openimages/dataset/blob/master/dict.csv
CLASS_LIST = ('/m/01g317','/m/04yx4')
img_name = "111111111111"
#download csv from https://storage.googleapis.com/openimages/web/download.html
with open('path\\train-annotations-bbox.csv', newline='') as csvfile:
bboxs = csv.reader(csvfile, delimiter=',', quotechar='|')
for bbox in bboxs:
if bbox[2] in CLASS_LIST:
if img_name != bbox[0]:
if not os.path.isfile("destination_path\\%s.jpg"%bbox[0]):
os.system("gsutil cp gs://open-images-dataset/train/%s.jpg destination_path"%bbox[0])
out_file = open("destination_path\\%s.txt"%bbox[0], 'w')
img_name = bbox[0]
if img_name == bbox[0]:
out_file.write(str(CLASS_LIST.index(bbox[2])) + " " + str(float(bbox[4])+(float(bbox[5])-float(bbox[4]))/2) + " " + str(float(bbox[6])+(float(bbox[7])-float(bbox[6]))/2)+ " " + str(float(bbox[5])-float(bbox[4])) + " " + str(float(bbox[7])-float(bbox[6])) + '\n')

View File

@@ -0,0 +1,19 @@
#!/bin/bash
#wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_bbox_val_v3.tgz
#other downloads: http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads
#read: https://pjreddie.com/darknet/imagenet/
mkdir -p labelled
wd=`pwd`
for f in val/*.xml;
do
label=`grep -m1 "<name>" $f | grep -oP '<name>\K[^<]*'`
im=`echo $f | sed 's/val/imgs/; s/xml/JPEG/'`
out=`echo $im | sed 's/JPEG/'${label}'.JPEG/; s/imgs/labelled/'`
ln -s ${wd}/$im ${wd}/$out
done
find ${wd}/labelled -name \*.JPEG > inet.val.list

View File

@@ -0,0 +1,141 @@
#!/usr/bin/env python
from __future__ import print_function
from __future__ import division
#
# This is a utility for converting ground truth data from the kitti format
# to the YOLO format.
#
#
#
# YOLO FORMAT
# .txt for each .jpg - in the same directory and with the same name
# <object-class> <x> <y> <width> <height>
#
# Where:
#
# <object-class> - integer number of object from 0 to (classes-1)
# <x> <y> <width> <height> - floats relative to image width/height 0.0 to 1.0
# eg. <x> = <absolute_x> / <image_width>
# Note: <x> <y> - are center of rectangle (not top-left corner)
#
# For example for img1.jpg you will be created img1.txt containing:
#
# 1 0.716797 0.395833 0.216406 0.147222
# 0 0.687109 0.379167 0.255469 0.158333
# 1 0.420312 0.395833 0.140625 0.166667
#
# KITTI FORMAT
#
# All images as .png in a separate folder to the .txt labels of the same name
# One label line is as follows:
#
# 1 type Describes the type of object: Car, Van, Truck,
# Pedestrian, Person_sitting, Cyclist, Tram,
# Misc or DontCare
# 1 truncated Float from 0 (non-truncated) to 1 (truncated), where
# truncated refers to the object leaving image boundaries
# 1 occluded Integer (0,1,2,3) indicating occlusion state:
# 0 = fully visible, 1 = partly occluded
# 2 = largely occluded, 3 = unknown
# 1 alpha Observation angle of object, ranging [-pi..pi]
# 4 bbox 2D bounding box of object in the image (0-based index):
# contains left, top, right, bottom pixel coordinates
# 3 dimensions 3D object dimensions: height, width, length (in meters)
# 3 location 3D object location x,y,z in camera coordinates (in meters)
# 1 rotation_y Rotation ry around Y-axis in camera coordinates [-pi..pi]
# 1 score Only for results: Float, indicating confidence in
# detection, needed for p/r curves, higher is better.
#
# Car 0.0 0 -1.5 57.0 17.3 614.1 200.12 1.65 1.67 3.64 -0.65 1.71 46.70 -1.59
# Cyclist 0.0 0 -2.46 665.45 160.00 717.9 217.9 1.7 0.4 1.6 2.4 1.3 22.1 -2.35
# Pedestrian 0.00 2 0.2 42.1 17.6 433.1 24.0 1.6 0.38 0.30 -5.8 1.6 23.1 -0.03
# DontCare -1 -1 -10 650.19 175.02 668.98 210.48 -1 -1 -1 -1000 -1000 -1000 -10
# core imports
import argparse
import sys
import os
import shutil
import cv2
kitti2yolotype_dict = {'Car': '0',
'Van': '0',
'Pedestrian': '1',
'Person_sitting': '1',
'Cyclist': '2',
'Truck': '3',
'Tram': '6',
'Misc': '6',
'DontCare': '6'}
def kitti2yolo(kitti_label, img_height, img_width):
kitti_label_arr = kitti_label.split(' ')
x1 = float(kitti_label_arr[4])
y1 = float(kitti_label_arr[5])
x2 = float(kitti_label_arr[6])
y2 = float(kitti_label_arr[7])
bb_width = x2 - x1
bb_height = y2 - y1
yolo_x = (x1 + 0.5*bb_width) / img_width
yolo_y = (y1 + 0.5*bb_height) / img_height
yolo_bb_width = bb_width / img_width
yolo_bb_height = bb_height / img_height
yolo_label = kitti2yolotype_dict[kitti_label_arr[0]]
return (yolo_label + ' '
+ str(yolo_x) + ' '
+ str(yolo_y) + ' '
+ str(yolo_bb_width) + ' '
+ str(yolo_bb_height))
def main(args):
# parse command line arguments
parser = argparse.ArgumentParser()
parser.add_argument("--kitti",
help="path to kitti-format images and labels, images\
should be under images_path/images and labels should\
be under images_path/labels")
parser.add_argument("--yolo",
help="path to output yolo-ready training data")
# kitti paths
args = parser.parse_args()
root_path = args.kitti
yolo_path = args.yolo
if root_path is None:
root_path = os.getcwd()
if (root_path[-1] != os.sep):
root_path += os.sep
kitti_images_path = root_path + 'image_2' + os.sep
kitti_labels_path = root_path + 'label_2' + os.sep
# yolo paths
if yolo_path is None:
yolo_path = root_path + 'yolo_labels' + os.sep
if not os.path.exists(yolo_path):
os.makedirs(yolo_path)
# load each kitti label, convert to yolo and save
for labelfilename in os.listdir(kitti_labels_path):
yolo_labels = []
with open(kitti_labels_path + labelfilename, 'r') as kittilabelfile:
cvimage = cv2.imread(kitti_images_path
+ labelfilename.split('.txt')[0] + '.png')
height, width, frame_depth = cvimage.shape
for kitti_label in kittilabelfile:
yolo_labels.append(kitti2yolo(kitti_label,
img_height=height,
img_width=width))
with open(yolo_path + labelfilename, 'w+') as yololabelfile:
for label in yolo_labels:
yololabelfile.write(label + '\n')
if __name__ == '__main__':
main(sys.argv)

View File

@@ -0,0 +1,391 @@
//usr/bin/cc -Ofast -lm "${0}" -o "${0%.c}" && ./"${0%.c}" "$@"; s=$?; rm ./"${0%.c}"; exit $s
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
typedef struct matrix{
int rows, cols;
double **vals;
} matrix;
matrix csv_to_matrix(char *filename, int header);
matrix make_matrix(int rows, int cols);
void zero_matrix(matrix m);
void copy(double *x, double *y, int n);
double dist(double *x, double *y, int n);
int *sample(int n);
int find_int_arg(int argc, char **argv, char *arg, int def);
int find_arg(int argc, char* argv[], char *arg);
int closest_center(double *datum, matrix centers)
{
int j;
int best = 0;
double best_dist = dist(datum, centers.vals[best], centers.cols);
for(j = 0; j < centers.rows; ++j){
double new_dist = dist(datum, centers.vals[j], centers.cols);
if(new_dist < best_dist){
best_dist = new_dist;
best = j;
}
}
return best;
}
double dist_to_closest_center(double *datum, matrix centers)
{
int ci = closest_center(datum, centers);
return dist(datum, centers.vals[ci], centers.cols);
}
int kmeans_expectation(matrix data, int *assignments, matrix centers)
{
int i;
int converged = 1;
for(i = 0; i < data.rows; ++i){
int closest = closest_center(data.vals[i], centers);
if(closest != assignments[i]) converged = 0;
assignments[i] = closest;
}
return converged;
}
void kmeans_maximization(matrix data, int *assignments, matrix centers)
{
int i,j;
int *counts = calloc(centers.rows, sizeof(int));
zero_matrix(centers);
for(i = 0; i < data.rows; ++i){
++counts[assignments[i]];
for(j = 0; j < data.cols; ++j){
centers.vals[assignments[i]][j] += data.vals[i][j];
}
}
for(i = 0; i < centers.rows; ++i){
if(counts[i]){
for(j = 0; j < centers.cols; ++j){
centers.vals[i][j] /= counts[i];
}
}
}
}
double WCSS(matrix data, int *assignments, matrix centers)
{
int i, j;
double sum = 0;
for(i = 0; i < data.rows; ++i){
int ci = assignments[i];
sum += (1 - dist(data.vals[i], centers.vals[ci], data.cols));
}
return sum / data.rows;
}
typedef struct{
int *assignments;
matrix centers;
} model;
void smart_centers(matrix data, matrix centers) {
int i,j;
copy(data.vals[rand()%data.rows], centers.vals[0], data.cols);
double *weights = calloc(data.rows, sizeof(double));
int clusters = centers.rows;
for (i = 1; i < clusters; ++i) {
double sum = 0;
centers.rows = i;
for (j = 0; j < data.rows; ++j) {
weights[j] = dist_to_closest_center(data.vals[j], centers);
sum += weights[j];
}
double r = sum*((double)rand()/RAND_MAX);
for (j = 0; j < data.rows; ++j) {
r -= weights[j];
if(r <= 0){
copy(data.vals[j], centers.vals[i], data.cols);
break;
}
}
}
free(weights);
}
void random_centers(matrix data, matrix centers){
int i;
int *s = sample(data.rows);
for(i = 0; i < centers.rows; ++i){
copy(data.vals[s[i]], centers.vals[i], data.cols);
}
free(s);
}
model do_kmeans(matrix data, int k)
{
matrix centers = make_matrix(k, data.cols);
int *assignments = calloc(data.rows, sizeof(int));
smart_centers(data, centers);
//random_centers(data, centers);
if(k == 1) kmeans_maximization(data, assignments, centers);
while(!kmeans_expectation(data, assignments, centers)){
kmeans_maximization(data, assignments, centers);
}
model m;
m.assignments = assignments;
m.centers = centers;
return m;
}
int main(int argc, char *argv[])
{
if(argc < 3){
fprintf(stderr, "usage: %s <csv-file> [points/centers/stats]\n", argv[0]);
return 0;
}
int i,j;
srand(time(0));
matrix data = csv_to_matrix(argv[1], 0);
int k = find_int_arg(argc, argv, "-k", 2);
int header = find_arg(argc, argv, "-h");
int count = find_arg(argc, argv, "-c");
if(strcmp(argv[2], "assignments")==0){
model m = do_kmeans(data, k);
int *assignments = m.assignments;
for(i = 0; i < k; ++i){
if(i != 0) printf("-\n");
for(j = 0; j < data.rows; ++j){
if(!(assignments[j] == i)) continue;
printf("%f, %f\n", data.vals[j][0], data.vals[j][1]);
}
}
}else if(strcmp(argv[2], "centers")==0){
model m = do_kmeans(data, k);
printf("WCSS: %f\n", WCSS(data, m.assignments, m.centers));
int *counts = 0;
if(count){
counts = calloc(k, sizeof(int));
for(j = 0; j < data.rows; ++j){
++counts[m.assignments[j]];
}
}
for(j = 0; j < m.centers.rows; ++j){
if(count) printf("%d, ", counts[j]);
printf("%f, %f\n", m.centers.vals[j][0], m.centers.vals[j][1]);
}
}else if(strcmp(argv[2], "scan")==0){
for(i = 1; i <= k; ++i){
model m = do_kmeans(data, i);
printf("%f\n", WCSS(data, m.assignments, m.centers));
}
}
return 0;
}
// Utility functions
int *sample(int n)
{
int i;
int *s = calloc(n, sizeof(int));
for(i = 0; i < n; ++i) s[i] = i;
for(i = n-1; i >= 0; --i){
int swap = s[i];
int index = rand()%(i+1);
s[i] = s[index];
s[index] = swap;
}
return s;
}
double dist(double *x, double *y, int n)
{
int i;
double mw = (x[0] < y[0]) ? x[0] : y[0];
double mh = (x[1] < y[1]) ? x[1] : y[1];
double inter = mw*mh;
double sum = x[0]*x[1] + y[0]*y[1];
double un = sum - inter;
double iou = inter/un;
return 1-iou;
}
void copy(double *x, double *y, int n)
{
int i;
for(i = 0; i < n; ++i) y[i] = x[i];
}
void error(char *s){
fprintf(stderr, "Error: %s\n", s);
exit(-1);
}
char *fgetl(FILE *fp)
{
if(feof(fp)) return 0;
int size = 512;
char *line = malloc(size*sizeof(char));
if(!fgets(line, size, fp)){
free(line);
return 0;
}
int curr = strlen(line);
while(line[curr-1]!='\n'){
size *= 2;
line = realloc(line, size*sizeof(char));
if(!line) error("Malloc");
fgets(&line[curr], size-curr, fp);
curr = strlen(line);
}
line[curr-1] = '\0';
return line;
}
// Matrix stuff
int count_fields(char *line)
{
int count = 0;
int done = 0;
char *c;
for(c = line; !done; ++c){
done = (*c == '\0');
if(*c == ',' || done) ++count;
}
return count;
}
double *parse_fields(char *l, int n)
{
int i;
double *field = calloc(n, sizeof(double));
for(i = 0; i < n; ++i){
field[i] = atof(l);
l = strchr(l, ',')+1;
}
return field;
}
matrix make_matrix(int rows, int cols)
{
matrix m;
m.rows = rows;
m.cols = cols;
m.vals = calloc(m.rows, sizeof(double *));
int i;
for(i = 0; i < m.rows; ++i) m.vals[i] = calloc(m.cols, sizeof(double));
return m;
}
void zero_matrix(matrix m)
{
int i, j;
for(i = 0; i < m.rows; ++i){
for(j = 0; j < m.cols; ++j) m.vals[i][j] = 0;
}
}
matrix csv_to_matrix(char *filename, int header)
{
FILE *fp = fopen(filename, "r");
if(!fp) error(filename);
matrix m;
m.cols = -1;
char *line;
int n = 0;
int size = 1024;
m.vals = calloc(size, sizeof(double*));
if(header) fgetl(fp);
while((line = fgetl(fp))){
if(m.cols == -1) m.cols = count_fields(line);
if(n == size){
size *= 2;
m.vals = realloc(m.vals, size*sizeof(double*));
}
m.vals[n] = parse_fields(line, m.cols);
free(line);
++n;
}
m.vals = realloc(m.vals, n*sizeof(double*));
m.rows = n;
return m;
}
// Argument parsing
void del_arg(int argc, char **argv, int index)
{
int i;
for(i = index; i < argc-1; ++i) argv[i] = argv[i+1];
argv[i] = 0;
}
int find_arg(int argc, char* argv[], char *arg)
{
int i;
for(i = 0; i < argc; ++i) {
if(!argv[i]) continue;
if(0==strcmp(argv[i], arg)) {
del_arg(argc, argv, i);
return 1;
}
}
return 0;
}
int find_int_arg(int argc, char **argv, char *arg, int def)
{
int i;
for(i = 0; i < argc-1; ++i){
if(!argv[i]) continue;
if(0==strcmp(argv[i], arg)){
def = atoi(argv[i+1]);
del_arg(argc, argv, i);
del_arg(argc, argv, i);
break;
}
}
return def;
}
float find_float_arg(int argc, char **argv, char *arg, float def)
{
int i;
for(i = 0; i < argc-1; ++i){
if(!argv[i]) continue;
if(0==strcmp(argv[i], arg)){
def = atof(argv[i+1]);
del_arg(argc, argv, i);
del_arg(argc, argv, i);
break;
}
}
return def;
}
char *find_char_arg(int argc, char **argv, char *arg, char *def)
{
int i;
for(i = 0; i < argc-1; ++i){
if(!argv[i]) continue;
if(0==strcmp(argv[i], arg)){
def = argv[i+1];
del_arg(argc, argv, i);
del_arg(argc, argv, i);
break;
}
}
return def;
}

View File

@@ -0,0 +1,113 @@
# -*- coding: utf-8 -*-
# @Time : 2018/4/25 20:28
# @Author : Adesun
# @Site : https://github.com/Adesun
# @File : log_parser.py
import argparse
import logging
import os
import platform
import re
import sys
# set non-interactive backend default when os is not windows
if sys.platform != 'win32':
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from matplotlib.ticker import MultipleLocator, FormatStrFormatter
def get_file_name_and_ext(filename):
(file_path, temp_filename) = os.path.split(filename)
(file_name, file_ext) = os.path.splitext(temp_filename)
return file_name, file_ext
def show_message(message, stop=False):
print(message)
if stop:
sys.exit(0)
def parse_args():
parser = argparse.ArgumentParser(description="training log parser by DeepKeeper ")
parser.add_argument('--source-dir', dest='source_dir', type=str, default='./',
help='the log source directory')
parser.add_argument('--save-dir', dest='save_dir', type=str, default='./',
help='the directory to be saved')
parser.add_argument('--csv-file', dest='csv_file', type=str, default="",
help='training log file')
parser.add_argument('--log-file', dest='log_file', type=str, default="",
help='training log file')
parser.add_argument('--show', dest='show_plot', type=bool, default=False,
help='whether to show')
return parser.parse_args()
def log_parser(args):
if not args.log_file:
show_message('log file must be specified.', True)
log_path = os.path.join(args.source_dir, args.log_file)
if not os.path.exists(log_path):
show_message('log file does not exist.', True)
file_name, _ = get_file_name_and_ext(log_path)
log_content = open(log_path).read()
iterations = []
losses = []
fig, ax = plt.subplots()
# set area we focus on
ax.set_ylim(0, 8)
major_locator = MultipleLocator()
minor_locator = MultipleLocator(0.5)
ax.yaxis.set_major_locator(major_locator)
ax.yaxis.set_minor_locator(minor_locator)
ax.yaxis.grid(True, which='minor')
pattern = re.compile(r"([\d].*): .*?, (.*?) avg")
# print(pattern.findall(log_content))
matches = pattern.findall(log_content)
# print(type(matches[0]))
counter = 0
log_count = len(matches)
if args.csv_file != '':
csv_path = os.path.join(args.save_dir, args.csv_file)
out_file = open(csv_path, 'w')
else:
csv_path = os.path.join(args.save_dir, file_name + '.csv')
out_file = open(csv_path, 'w')
for match in matches:
counter += 1
if log_count > 200:
if counter % 200 == 0:
print('parsing {}/{}'.format(counter, log_count))
else:
print('parsing {}/{}'.format(counter, log_count))
iteration, loss = match
iterations.append(int(iteration))
losses.append(float(loss))
out_file.write(iteration + ',' + loss + '\n')
ax.plot(iterations, losses)
plt.xlabel('Iteration')
plt.ylabel('Loss')
plt.tight_layout()
# saved as svg
save_path = os.path.join(args.save_dir, file_name + '.svg')
plt.savefig(save_path, dpi=300, format="svg")
if args.show_plot:
plt.show()
if __name__ == "__main__":
args = parse_args()
log_parser(args)

View File

@@ -0,0 +1,17 @@
# parsing training log and plot
## Requirements
1. matplotlib
## Usage
1. --source-dir the directory of training log files
2. --save-dir the directory to save loss curve, image and csv file
3. --log-file log file name to be parsed
4. --csv-file csv file name to save loss data, default it's same with training log file name
5. --show whether to show after finished parsing, default False, just works on windows or linux with GUI desktop
`python log_parser.py --source-dir ./ --save-dir ./ --log-file test.log --show true`
![plot](https://github.com/AlexeyAB/darknet/blob/master/scripts/log_parser/test_new.svg)

File diff suppressed because it is too large Load Diff

After

Width:  |  Height:  |  Size: 76 KiB

View File

@@ -0,0 +1,4 @@
#!/usr/bin/env bash
export CUDA_VERSION="12.2"
export CUDA_VERSION_DASHED="${CUDA_VERSION//./-}"

View File

@@ -0,0 +1,101 @@
#!/usr/bin/env python
# Adapt from ->
# --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------
# <- Written by Yaping Sun
"""Reval = re-eval. Re-evaluate saved detections."""
import os, sys, argparse
import numpy as np
import cPickle
from voc_eval import voc_eval
def parse_args():
"""
Parse input arguments
"""
parser = argparse.ArgumentParser(description='Re-evaluate results')
parser.add_argument('output_dir', nargs=1, help='results directory',
type=str)
parser.add_argument('--voc_dir', dest='voc_dir', default='data/VOCdevkit', type=str)
parser.add_argument('--year', dest='year', default='2017', type=str)
parser.add_argument('--image_set', dest='image_set', default='test', type=str)
parser.add_argument('--classes', dest='class_file', default='data/voc.names', type=str)
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
return args
def get_voc_results_file_template(image_set, out_dir = 'results'):
filename = 'comp4_det_' + image_set + '_{:s}.txt'
path = os.path.join(out_dir, filename)
return path
def do_python_eval(devkit_path, year, image_set, classes, output_dir = 'results'):
annopath = os.path.join(
devkit_path,
'VOC' + year,
'Annotations',
'{:s}.xml')
imagesetfile = os.path.join(
devkit_path,
'VOC' + year,
'ImageSets',
'Main',
image_set + '.txt')
cachedir = os.path.join(devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(year) < 2010 else False
print 'VOC07 metric? ' + ('Yes' if use_07_metric else 'No')
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for i, cls in enumerate(classes):
if cls == '__background__':
continue
filename = get_voc_results_file_template(image_set).format(cls)
rec, prec, ap = voc_eval(
filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
print('AP for {} = {:.4f}'.format(cls, ap))
with open(os.path.join(output_dir, cls + '_pr.pkl'), 'w') as f:
cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print('Mean AP = {:.4f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('Results:')
for ap in aps:
print('{:.3f}'.format(ap))
print('{:.3f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
if __name__ == '__main__':
args = parse_args()
output_dir = os.path.abspath(args.output_dir[0])
with open(args.class_file, 'r') as f:
lines = f.readlines()
classes = [t.strip('\n') for t in lines]
print 'Evaluating detections'
do_python_eval(args.voc_dir, args.year, args.image_set, classes, output_dir)

View File

@@ -0,0 +1,104 @@
#!/usr/bin/env python
# Adapt from ->
# --------------------------------------------------------
# Fast R-CNN
# Copyright (c) 2015 Microsoft
# Licensed under The MIT License [see LICENSE for details]
# Written by Ross Girshick
# --------------------------------------------------------
# <- Written by Yaping Sun
"""Reval = re-eval. Re-evaluate saved detections."""
import os, sys, argparse
import numpy as np
import _pickle as cPickle
#import cPickle
from voc_eval_py3 import voc_eval
def parse_args():
"""
Parse input arguments
"""
parser = argparse.ArgumentParser(description='Re-evaluate results')
parser.add_argument('output_dir', nargs=1, help='results directory',
type=str)
parser.add_argument('--voc_dir', dest='voc_dir', default='data/VOCdevkit', type=str)
parser.add_argument('--year', dest='year', default='2017', type=str)
parser.add_argument('--image_set', dest='image_set', default='test', type=str)
parser.add_argument('--classes', dest='class_file', default='data/voc.names', type=str)
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args()
return args
def get_voc_results_file_template(image_set, out_dir = 'results'):
filename = 'comp4_det_' + image_set + '_{:s}.txt'
path = os.path.join(out_dir, filename)
return path
def do_python_eval(devkit_path, year, image_set, classes, output_dir = 'results'):
annopath = os.path.join(
devkit_path,
'VOC' + year,
'Annotations',
'{}.xml')
imagesetfile = os.path.join(
devkit_path,
'VOC' + year,
'ImageSets',
'Main',
image_set + '.txt')
cachedir = os.path.join(devkit_path, 'annotations_cache')
aps = []
# The PASCAL VOC metric changed in 2010
use_07_metric = True if int(year) < 2010 else False
print('VOC07 metric? ' + ('Yes' if use_07_metric else 'No'))
print('devkit_path=',devkit_path,', year = ',year)
if not os.path.isdir(output_dir):
os.mkdir(output_dir)
for i, cls in enumerate(classes):
if cls == '__background__':
continue
filename = get_voc_results_file_template(image_set).format(cls)
rec, prec, ap = voc_eval(
filename, annopath, imagesetfile, cls, cachedir, ovthresh=0.5,
use_07_metric=use_07_metric)
aps += [ap]
print('AP for {} = {:.4f}'.format(cls, ap))
with open(os.path.join(output_dir, cls + '_pr.pkl'), 'wb') as f:
cPickle.dump({'rec': rec, 'prec': prec, 'ap': ap}, f)
print('Mean AP = {:.4f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('Results:')
for ap in aps:
print('{:.3f}'.format(ap))
print('{:.3f}'.format(np.mean(aps)))
print('~~~~~~~~')
print('')
print('--------------------------------------------------------------')
print('Results computed with the **unofficial** Python eval code.')
print('Results should be very close to the official MATLAB eval code.')
print('-- Thanks, The Management')
print('--------------------------------------------------------------')
if __name__ == '__main__':
args = parse_args()
output_dir = os.path.abspath(args.output_dir[0])
with open(args.class_file, 'r') as f:
lines = f.readlines()
classes = [t.strip('\n') for t in lines]
print('Evaluating detections')
do_python_eval(args.voc_dir, args.year, args.image_set, classes, output_dir)

View File

@@ -0,0 +1,41 @@
#!/usr/bin/env pwsh
param (
[switch]$InstallCUDA = $false
)
Import-Module -Name $PSScriptRoot/utils.psm1 -Force
if ($null -eq (Get-Command "choco.exe" -ErrorAction SilentlyContinue)) {
# Download and install Chocolatey
Invoke-Expression ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))
Throw "Please close and re-open powershell and then re-run setup.ps1 script"
}
Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y cmake ninja powershell git vscode"
Start-Process -FilePath "choco" -Verb runAs -ArgumentList " install -y visualstudio2022buildtools --package-parameters `"--add Microsoft.VisualStudio.Component.VC.CoreBuildTools --includeRecommended --includeOptional --passive --locale en-US --lang en-US`""
Push-Location $PSScriptRoot
if ($InstallCUDA) {
& $PSScriptRoot/deploy-cuda.ps1
$env:CUDA_PATH = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v${cuda_version_short}"
$env:CUDA_TOOLKIT_ROOT_DIR = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v${cuda_version_short}"
$env:CUDACXX = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v${cuda_version_short}\\bin\\nvcc.exe"
$CUDAisAvailable = $true
}
else {
if (-not $null -eq $env:CUDA_PATH) {
$CUDAisAvailable = $true
}
else{
$CUDAisAvailable = $false
}
}
if ($CUDAisAvailable) {
& $PSScriptRoot/../build.ps1 -UseVCPKG -ForceLocalVCPKG -EnableOPENCV -EnableCUDA -DisableInteractive -DoNotUpdateTOOL
#& $PSScriptRoot/../build.ps1 -UseVCPKG -EnableOPENCV -EnableCUDA -EnableOPENCV_CUDA -DisableInteractive -DoNotUpdateTOOL
}
else {
& $PSScriptRoot/../build.ps1 -UseVCPKG -EnableOPENCV -DisableInteractive -DoNotUpdateTOOL
}

View File

@@ -0,0 +1,113 @@
#!/usr/bin/env bash
install_tools=false
install_cuda=false
bypass_driver_installation=false
POSITIONAL=()
while [[ $# -gt 0 ]]
do
key="$1"
case $key in
-InstallCUDA|--InstallCUDA)
install_cuda=true
shift
;;
-InstallTOOLS|--InstallTOOLS)
install_tools=true
shift
;;
-BypassDRIVER|--BypassDRIVER)
bypass_driver_installation=true
shift
;;
*) # unknown option
POSITIONAL+=("$1") # save it in an array for later
shift # past argument
;;
esac
done
set -- "${POSITIONAL[@]}" # restore positional parameters
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )"
echo "This script is located in $script_dir"
cd $script_dir/..
temp_folder="./temp"
mkdir -p $temp_folder
cd $temp_folder
if [ -f $script_dir/requested_cuda_version.sh ]; then
echo "Loading $script_dir/requested_cuda_version.sh"
source $script_dir/requested_cuda_version.sh
else
echo "Unable to find requested_cuda_version.sh script"
exit 1
fi
if [[ "$OSTYPE" == "darwin"* ]]; then
if [ "$install_cuda" = true ] ; then
echo "Unable to install CUDA on macOS, please wait for a future script update or do not put -InstallCUDA command line flag to continue"
exit 2
fi
if [ "$install_tools" = true ] ; then
echo "Unable to provide tools on macOS, please wait for a future script update or do not put -InstallTOOLS command line flag to continue"
exit 3
fi
elif [[ $(cut -f2 <<< $(lsb_release -i)) == "Ubuntu" ]]; then
echo "Running in $(cut -f2 <<< $(lsb_release -i))"
echo "InstallCUDA: $install_cuda"
echo "InstallTOOLS: $install_tools"
if [ "$install_cuda" = true ] ; then
echo "Running $script_dir/deploy-cuda.sh"
$script_dir/deploy-cuda.sh
if [ "$bypass_driver_installation" = true ] ; then
sudo ln -s /usr/local/cuda-${CUDA_VERSION}/lib64/stubs/libcuda.so /usr/local/cuda-${CUDA_VERSION}/lib64/stubs/libcuda.so.1
sudo ln -s /usr/local/cuda-${CUDA_VERSION}/lib64/stubs/libcuda.so /usr/local/cuda-${CUDA_VERSION}/lib64/libcuda.so.1
sudo ln -s /usr/local/cuda-${CUDA_VERSION}/lib64/stubs/libcuda.so /usr/local/cuda-${CUDA_VERSION}/lib64/libcuda.so
fi
export PATH=/usr/local/cuda/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda/lib64:$LD_LIBRARY_PATH
export CUDACXX=/usr/local/cuda/bin/nvcc
export CUDA_PATH=/usr/local/cuda
export CUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda
fi
if [ "$install_tools" = true ] ; then
echo "Installing tools"
sudo apt-get update
sudo apt-get install -y --no-install-recommends git ninja-build build-essential g++ nasm yasm gperf
sudo apt-get install -y --no-install-recommends apt-transport-https ca-certificates gnupg software-properties-common wget
sudo apt-get install -y --no-install-recommends libgles2-mesa-dev libx11-dev libxft-dev libxext-dev libxrandr-dev libxi-dev libxcursor-dev libxdamage-dev libxinerama-dev libdbus-1-dev libxtst-dev
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | sudo tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null
sudo apt-add-repository "deb https://apt.kitware.com/ubuntu/ $(cut -f2 <<< $(lsb_release -c)) main"
wget -q https://packages.microsoft.com/config/ubuntu/$(cut -f2 <<< $(lsb_release -r))/packages-microsoft-prod.deb
sudo dpkg -i packages-microsoft-prod.deb
sudo add-apt-repository universe
sudo apt-get update
sudo apt-get dist-upgrade -y
sudo apt-get install -y --no-install-recommends cmake
sudo apt-get install -y --no-install-recommends powershell
sudo apt-get install -y --no-install-recommends curl zip unzip tar
sudo apt-get install -y --no-install-recommends pkg-config autoconf libtool bison
sudo apt-get clean
fi
else
if [ "$install_cuda" = true ] ; then
echo "Unable to install CUDA on this OS, please wait for a future script update or do not put -InstallCUDA command line flag to continue"
exit 4
fi
if [ "$install_tools" = true ] ; then
echo "Unable to install tools on this OS, please wait for a future script update or do not put -InstallTOOLS command line flag to continue"
exit 5
fi
fi
cd ..
rm -rf "$temp_folder"
echo "Building darknet"
if [[ -v CUDA_PATH ]]; then
./build.ps1 -UseVCPKG -EnableOPENCV -EnableCUDA -EnableCUDNN -DisableInteractive -DoNotUpdateTOOL
#./build.ps1 -UseVCPKG -EnableOPENCV -EnableCUDA -EnableCUDNN -EnableOPENCV_CUDA -DisableInteractive -DoNotUpdateTOOL
else
./build.ps1 -UseVCPKG -EnableOPENCV -DisableInteractive -DoNotUpdateTOOL
fi

View File

@@ -0,0 +1,342 @@
<#
Copyright (c) Stefano Sinigardi
MIT License
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
#>
$utils_psm1_version = "0.3.0"
$IsWindowsPowerShell = switch ( $PSVersionTable.PSVersion.Major ) {
5 { $true }
4 { $true }
3 { $true }
2 { $true }
default { $false }
}
$ExecutableSuffix = ""
if ($IsWindowsPowerShell -or $IsWindows) {
$ExecutableSuffix = ".exe"
}
$64bitPwsh = $([Environment]::Is64BitProcess)
$64bitOS = $([Environment]::Is64BitOperatingSystem)
Push-Location $PSScriptRoot
$GIT_EXE = Get-Command "git" -ErrorAction SilentlyContinue | Select-Object -ExpandProperty Definition
if ($GIT_EXE) {
$IsInGitSubmoduleString = $(git rev-parse --show-superproject-working-tree 2> $null)
if ($IsInGitSubmoduleString.Length -eq 0) {
$IsInGitSubmodule = $false
}
else {
$IsInGitSubmodule = $true
}
}
else {
$IsInGitSubmodule = $false
}
Pop-Location
$cuda_version_full = "12.2.0"
$cuda_version_short = "12.2"
$cuda_version_full_dashed = $cuda_version_full.replace('.', '-')
$cuda_version_short_dashed = $cuda_version_short.replace('.', '-')
function getProgramFiles32bit() {
$out = ${env:PROGRAMFILES(X86)}
if ($null -eq $out) {
$out = ${env:PROGRAMFILES}
}
if ($null -eq $out) {
MyThrow("Could not find [Program Files 32-bit]")
}
return $out
}
function getLatestVisualStudioWithDesktopWorkloadPath([bool]$required = $true) {
$programFiles = getProgramFiles32bit
$vswhereExe = "$programFiles\Microsoft Visual Studio\Installer\vswhere.exe"
if (Test-Path $vswhereExe) {
$output = & $vswhereExe -products * -latest -requires Microsoft.VisualStudio.Workload.NativeDesktop -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationPath = $instance.InstallationPath -replace "\\$" # Remove potential trailing backslash
}
if (!$installationPath) {
#Write-Host "Warning: no full Visual Studio setup has been found, extending search to include also partial installations" -ForegroundColor Yellow
$output = & $vswhereExe -products * -latest -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationPath = $instance.InstallationPath -replace "\\$" # Remove potential trailing backslash
}
}
if (!$installationPath) {
#Write-Host "Warning: no full Visual Studio setup has been found, extending search to include also pre-release installations" -ForegroundColor Yellow
$output = & $vswhereExe -prerelease -products * -latest -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationPath = $instance.InstallationPath -replace "\\$" # Remove potential trailing backslash
}
}
if (!$installationPath) {
if ($required) {
MyThrow("Could not locate any installation of Visual Studio")
}
else {
Write-Host "Could not locate any installation of Visual Studio" -ForegroundColor Red
return $null
}
}
}
else {
if ($required) {
MyThrow("Could not locate vswhere at $vswhereExe")
}
else {
Write-Host "Could not locate vswhere at $vswhereExe" -ForegroundColor Red
return $null
}
}
return $installationPath
}
function getLatestVisualStudioWithDesktopWorkloadVersion([bool]$required = $true) {
$programFiles = getProgramFiles32bit
$vswhereExe = "$programFiles\Microsoft Visual Studio\Installer\vswhere.exe"
if (Test-Path $vswhereExe) {
$output = & $vswhereExe -products * -latest -requires Microsoft.VisualStudio.Workload.NativeDesktop -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationVersion = $instance.InstallationVersion
}
if (!$installationVersion) {
#Write-Host "Warning: no full Visual Studio setup has been found, extending search to include also partial installations" -ForegroundColor Yellow
$output = & $vswhereExe -products * -latest -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationVersion = $instance.installationVersion
}
}
if (!$installationVersion) {
#Write-Host "Warning: no full Visual Studio setup has been found, extending search to include also pre-release installations" -ForegroundColor Yellow
$output = & $vswhereExe -prerelease -products * -latest -format xml
[xml]$asXml = $output
foreach ($instance in $asXml.instances.instance) {
$installationVersion = $instance.installationVersion
}
}
if (!$installationVersion) {
if ($required) {
MyThrow("Could not locate any installation of Visual Studio")
}
else {
Write-Host "Could not locate any installation of Visual Studio" -ForegroundColor Red
return $null
}
}
}
else {
if ($required) {
MyThrow("Could not locate vswhere at $vswhereExe")
}
else {
Write-Host "Could not locate vswhere at $vswhereExe" -ForegroundColor Red
return $null
}
}
return $installationVersion
}
function DownloadNinja() {
Write-Host "Downloading a portable version of Ninja" -ForegroundColor Yellow
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue ninja
Remove-Item -Force -ErrorAction SilentlyContinue ninja.zip
if ($IsWindows -or $IsWindowsPowerShell) {
$url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-win.zip"
}
elseif ($IsLinux) {
$url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-linux.zip"
}
elseif ($IsMacOS) {
$url = "https://github.com/ninja-build/ninja/releases/download/v1.10.2/ninja-mac.zip"
}
else {
MyThrow("Unknown OS, unsupported")
}
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile "ninja.zip"
Expand-Archive -Path ninja.zip
Remove-Item -Force -ErrorAction SilentlyContinue ninja.zip
return "./ninja${ExecutableSuffix}"
}
function DownloadAria2() {
Write-Host "Downloading a portable version of Aria2" -ForegroundColor Yellow
if ($IsWindows -or $IsWindowsPowerShell) {
$basename = "aria2-1.35.0-win-32bit-build1"
$zipName = "${basename}.zip"
$outFolder = "$basename/$basename"
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://github.com/aria2/aria2/releases/download/release-1.35.0/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
Expand-Archive -Path $zipName
}
elseif ($IsLinux) {
$basename = "aria2-1.36.0-linux-gnu-64bit-build1"
$zipName = "${basename}.tar.bz2"
$outFolder = $basename
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://github.com/q3aql/aria2-static-builds/releases/download/v1.36.0/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
tar xf $zipName
}
elseif ($IsMacOS) {
$basename = "aria2-1.35.0-osx-darwin"
$zipName = "${basename}.tar.bz2"
$outFolder = "aria2-1.35.0/bin"
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://github.com/aria2/aria2/releases/download/release-1.35.0/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
tar xf $zipName
}
else {
MyThrow("Unknown OS, unsupported")
}
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
return "./$outFolder/aria2c${ExecutableSuffix}"
}
function Download7Zip() {
Write-Host "Downloading a portable version of 7-Zip" -ForegroundColor Yellow
if ($IsWindows -or $IsWindowsPowerShell) {
$basename = "7za920"
$zipName = "${basename}.zip"
$outFolder = "$basename"
$outSuffix = "a"
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://www.7-zip.org/a/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
Expand-Archive -Path $zipName
}
elseif ($IsLinux) {
$basename = "7z2201-linux-x64"
$zipName = "${basename}.tar.xz"
$outFolder = $basename
$outSuffix = "z"
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://www.7-zip.org/a/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
tar xf $zipName
}
elseif ($IsMacOS) {
$basename = "7z2107-mac"
$zipName = "${basename}.tar.xz"
$outFolder = $basename
$outSuffix = "z"
Remove-Item -Force -Recurse -ErrorAction SilentlyContinue $outFolder
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
$url = "https://www.7-zip.org/a/$zipName"
Invoke-RestMethod -Uri $url -Method Get -ContentType application/zip -OutFile $zipName
tar xf $zipName
}
else {
MyThrow("Unknown OS, unsupported")
}
Remove-Item -Force -ErrorAction SilentlyContinue $zipName
return "./$outFolder/7z${outSuffix}${ExecutableSuffix}"
}
Function MyThrow ($Message) {
if ($global:DisableInteractive) {
Write-Host $Message -ForegroundColor Red
throw
}
else {
# Check if running in PowerShell ISE
if ($psISE) {
# "ReadKey" not supported in PowerShell ISE.
# Show MessageBox UI
$Shell = New-Object -ComObject "WScript.Shell"
$Shell.Popup($Message, 0, "OK", 0)
throw
}
$Ignore =
16, # Shift (left or right)
17, # Ctrl (left or right)
18, # Alt (left or right)
20, # Caps lock
91, # Windows key (left)
92, # Windows key (right)
93, # Menu key
144, # Num lock
145, # Scroll lock
166, # Back
167, # Forward
168, # Refresh
169, # Stop
170, # Search
171, # Favorites
172, # Start/Home
173, # Mute
174, # Volume Down
175, # Volume Up
176, # Next Track
177, # Previous Track
178, # Stop Media
179, # Play
180, # Mail
181, # Select Media
182, # Application 1
183 # Application 2
Write-Host $Message -ForegroundColor Red
Write-Host -NoNewline "Press any key to continue..."
while (($null -eq $KeyInfo.VirtualKeyCode) -or ($Ignore -contains $KeyInfo.VirtualKeyCode)) {
$KeyInfo = $Host.UI.RawUI.ReadKey("NoEcho, IncludeKeyDown")
}
Write-Host ""
throw
}
}
Export-ModuleMember -Variable utils_psm1_version
Export-ModuleMember -Variable IsWindowsPowerShell
Export-ModuleMember -Variable IsInGitSubmodule
Export-ModuleMember -Variable 64bitPwsh
Export-ModuleMember -Variable 64bitOS
Export-ModuleMember -Variable cuda_version_full
Export-ModuleMember -Variable cuda_version_short
Export-ModuleMember -Variable cuda_version_full_dashed
Export-ModuleMember -Variable cuda_version_short_dashed
Export-ModuleMember -Function getProgramFiles32bit
Export-ModuleMember -Function getLatestVisualStudioWithDesktopWorkloadPath
Export-ModuleMember -Function getLatestVisualStudioWithDesktopWorkloadVersion
Export-ModuleMember -Function DownloadNinja
Export-ModuleMember -Function DownloadAria2
Export-ModuleMember -Function Download7Zip
Export-ModuleMember -Function MyThrow

View File

@@ -0,0 +1,200 @@
# --------------------------------------------------------
# Fast/er R-CNN
# Licensed under The MIT License [see LICENSE for details]
# Written by Bharath Hariharan
# --------------------------------------------------------
import xml.etree.ElementTree as ET
import os
import cPickle
import numpy as np
def parse_rec(filename):
""" Parse a PASCAL VOC xml file """
tree = ET.parse(filename)
objects = []
for obj in tree.findall('object'):
obj_struct = {}
obj_struct['name'] = obj.find('name').text
#obj_struct['pose'] = obj.find('pose').text
#obj_struct['truncated'] = int(obj.find('truncated').text)
obj_struct['difficult'] = int(obj.find('difficult').text)
bbox = obj.find('bndbox')
obj_struct['bbox'] = [int(bbox.find('xmin').text),
int(bbox.find('ymin').text),
int(bbox.find('xmax').text),
int(bbox.find('ymax').text)]
objects.append(obj_struct)
return objects
def voc_ap(rec, prec, use_07_metric=False):
""" ap = voc_ap(rec, prec, [use_07_metric])
Compute VOC AP given precision and recall.
If use_07_metric is true, uses the
VOC 07 11 point method (default:False).
"""
if use_07_metric:
# 11 point metric
ap = 0.
for t in np.arange(0., 1.1, 0.1):
if np.sum(rec >= t) == 0:
p = 0
else:
p = np.max(prec[rec >= t])
ap = ap + p / 11.
else:
# correct AP calculation
# first append sentinel values at the end
mrec = np.concatenate(([0.], rec, [1.]))
mpre = np.concatenate(([0.], prec, [0.]))
# compute the precision envelope
for i in range(mpre.size - 1, 0, -1):
mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
# to calculate area under PR curve, look for points
# where X axis (recall) changes value
i = np.where(mrec[1:] != mrec[:-1])[0]
# and sum (\Delta recall) * prec
ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
return ap
def voc_eval(detpath,
annopath,
imagesetfile,
classname,
cachedir,
ovthresh=0.5,
use_07_metric=False):
"""rec, prec, ap = voc_eval(detpath,
annopath,
imagesetfile,
classname,
[ovthresh],
[use_07_metric])
Top level function that does the PASCAL VOC evaluation.
detpath: Path to detections
detpath.format(classname) should produce the detection results file.
annopath: Path to annotations
annopath.format(imagename) should be the xml annotations file.
imagesetfile: Text file containing the list of images, one image per line.
classname: Category name (duh)
cachedir: Directory for caching the annotations
[ovthresh]: Overlap threshold (default = 0.5)
[use_07_metric]: Whether to use VOC07's 11 point AP computation
(default False)
"""
# assumes detections are in detpath.format(classname)
# assumes annotations are in annopath.format(imagename)
# assumes imagesetfile is a text file with each line an image name
# cachedir caches the annotations in a pickle file
# first load gt
if not os.path.isdir(cachedir):
os.mkdir(cachedir)
cachefile = os.path.join(cachedir, 'annots.pkl')
# read list of images
with open(imagesetfile, 'r') as f:
lines = f.readlines()
imagenames = [x.strip() for x in lines]
if not os.path.isfile(cachefile):
# load annots
recs = {}
for i, imagename in enumerate(imagenames):
recs[imagename] = parse_rec(annopath.format(imagename))
if i % 100 == 0:
print 'Reading annotation for {:d}/{:d}'.format(
i + 1, len(imagenames))
# save
print 'Saving cached annotations to {:s}'.format(cachefile)
with open(cachefile, 'w') as f:
cPickle.dump(recs, f)
else:
# load
with open(cachefile, 'r') as f:
recs = cPickle.load(f)
# extract gt objects for this class
class_recs = {}
npos = 0
for imagename in imagenames:
R = [obj for obj in recs[imagename] if obj['name'] == classname]
bbox = np.array([x['bbox'] for x in R])
difficult = np.array([x['difficult'] for x in R]).astype(np.bool)
det = [False] * len(R)
npos = npos + sum(~difficult)
class_recs[imagename] = {'bbox': bbox,
'difficult': difficult,
'det': det}
# read dets
detfile = detpath.format(classname)
with open(detfile, 'r') as f:
lines = f.readlines()
splitlines = [x.strip().split(' ') for x in lines]
image_ids = [x[0] for x in splitlines]
confidence = np.array([float(x[1]) for x in splitlines])
BB = np.array([[float(z) for z in x[2:]] for x in splitlines])
# sort by confidence
sorted_ind = np.argsort(-confidence)
sorted_scores = np.sort(-confidence)
BB = BB[sorted_ind, :]
image_ids = [image_ids[x] for x in sorted_ind]
# go down dets and mark TPs and FPs
nd = len(image_ids)
tp = np.zeros(nd)
fp = np.zeros(nd)
for d in range(nd):
R = class_recs[image_ids[d]]
bb = BB[d, :].astype(float)
ovmax = -np.inf
BBGT = R['bbox'].astype(float)
if BBGT.size > 0:
# compute overlaps
# intersection
ixmin = np.maximum(BBGT[:, 0], bb[0])
iymin = np.maximum(BBGT[:, 1], bb[1])
ixmax = np.minimum(BBGT[:, 2], bb[2])
iymax = np.minimum(BBGT[:, 3], bb[3])
iw = np.maximum(ixmax - ixmin + 1., 0.)
ih = np.maximum(iymax - iymin + 1., 0.)
inters = iw * ih
# union
uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
(BBGT[:, 2] - BBGT[:, 0] + 1.) *
(BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)
overlaps = inters / uni
ovmax = np.max(overlaps)
jmax = np.argmax(overlaps)
if ovmax > ovthresh:
if not R['difficult'][jmax]:
if not R['det'][jmax]:
tp[d] = 1.
R['det'][jmax] = 1
else:
fp[d] = 1.
else:
fp[d] = 1.
# compute precision recall
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(npos)
# avoid divide by zero in case the first detection matches a difficult
# ground truth
prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, use_07_metric)
return rec, prec, ap

View File

@@ -0,0 +1,201 @@
# --------------------------------------------------------
# Fast/er R-CNN
# Licensed under The MIT License [see LICENSE for details]
# Written by Bharath Hariharan
# --------------------------------------------------------
import xml.etree.ElementTree as ET
import os
#import cPickle
import _pickle as cPickle
import numpy as np
def parse_rec(filename):
""" Parse a PASCAL VOC xml file """
tree = ET.parse(filename)
objects = []
for obj in tree.findall('object'):
obj_struct = {}
obj_struct['name'] = obj.find('name').text
#obj_struct['pose'] = obj.find('pose').text
#obj_struct['truncated'] = int(obj.find('truncated').text)
obj_struct['difficult'] = int(obj.find('difficult').text)
bbox = obj.find('bndbox')
obj_struct['bbox'] = [int(bbox.find('xmin').text),
int(bbox.find('ymin').text),
int(bbox.find('xmax').text),
int(bbox.find('ymax').text)]
objects.append(obj_struct)
return objects
def voc_ap(rec, prec, use_07_metric=False):
""" ap = voc_ap(rec, prec, [use_07_metric])
Compute VOC AP given precision and recall.
If use_07_metric is true, uses the
VOC 07 11 point method (default:False).
"""
if use_07_metric:
# 11 point metric
ap = 0.
for t in np.arange(0., 1.1, 0.1):
if np.sum(rec >= t) == 0:
p = 0
else:
p = np.max(prec[rec >= t])
ap = ap + p / 11.
else:
# correct AP calculation
# first append sentinel values at the end
mrec = np.concatenate(([0.], rec, [1.]))
mpre = np.concatenate(([0.], prec, [0.]))
# compute the precision envelope
for i in range(mpre.size - 1, 0, -1):
mpre[i - 1] = np.maximum(mpre[i - 1], mpre[i])
# to calculate area under PR curve, look for points
# where X axis (recall) changes value
i = np.where(mrec[1:] != mrec[:-1])[0]
# and sum (\Delta recall) * prec
ap = np.sum((mrec[i + 1] - mrec[i]) * mpre[i + 1])
return ap
def voc_eval(detpath,
annopath,
imagesetfile,
classname,
cachedir,
ovthresh=0.5,
use_07_metric=False):
"""rec, prec, ap = voc_eval(detpath,
annopath,
imagesetfile,
classname,
[ovthresh],
[use_07_metric])
Top level function that does the PASCAL VOC evaluation.
detpath: Path to detections
detpath.format(classname) should produce the detection results file.
annopath: Path to annotations
annopath.format(imagename) should be the xml annotations file.
imagesetfile: Text file containing the list of images, one image per line.
classname: Category name (duh)
cachedir: Directory for caching the annotations
[ovthresh]: Overlap threshold (default = 0.5)
[use_07_metric]: Whether to use VOC07's 11 point AP computation
(default False)
"""
# assumes detections are in detpath.format(classname)
# assumes annotations are in annopath.format(imagename)
# assumes imagesetfile is a text file with each line an image name
# cachedir caches the annotations in a pickle file
# first load gt
if not os.path.isdir(cachedir):
os.mkdir(cachedir)
cachefile = os.path.join(cachedir, 'annots.pkl')
# read list of images
with open(imagesetfile, 'r') as f:
lines = f.readlines()
imagenames = [x.strip() for x in lines]
if not os.path.isfile(cachefile):
# load annots
recs = {}
for i, imagename in enumerate(imagenames):
recs[imagename] = parse_rec(annopath.format(imagename))
#if i % 100 == 0:
#print('Reading annotation for {:d}/{:d}').format(i + 1, len(imagenames))
# save
#print('Saving cached annotations to {:s}').format(cachefile)
with open(cachefile, 'wb') as f:
cPickle.dump(recs, f)
else:
# load
print('!!! cachefile = ',cachefile)
with open(cachefile, 'rb') as f:
recs = cPickle.load(f)
# extract gt objects for this class
class_recs = {}
npos = 0
for imagename in imagenames:
R = [obj for obj in recs[imagename] if obj['name'] == classname]
bbox = np.array([x['bbox'] for x in R])
difficult = np.array([x['difficult'] for x in R]).astype(np.bool)
det = [False] * len(R)
npos = npos + sum(~difficult)
class_recs[imagename] = {'bbox': bbox,
'difficult': difficult,
'det': det}
# read dets
detfile = detpath.format(classname)
with open(detfile, 'r') as f:
lines = f.readlines()
splitlines = [x.strip().split(' ') for x in lines]
image_ids = [x[0] for x in splitlines]
confidence = np.array([float(x[1]) for x in splitlines])
BB = np.array([[float(z) for z in x[2:]] for x in splitlines])
# sort by confidence
sorted_ind = np.argsort(-confidence)
sorted_scores = np.sort(-confidence)
BB = BB[sorted_ind, :]
image_ids = [image_ids[x] for x in sorted_ind]
# go down dets and mark TPs and FPs
nd = len(image_ids)
tp = np.zeros(nd)
fp = np.zeros(nd)
for d in range(nd):
R = class_recs[image_ids[d]]
bb = BB[d, :].astype(float)
ovmax = -np.inf
BBGT = R['bbox'].astype(float)
if BBGT.size > 0:
# compute overlaps
# intersection
ixmin = np.maximum(BBGT[:, 0], bb[0])
iymin = np.maximum(BBGT[:, 1], bb[1])
ixmax = np.minimum(BBGT[:, 2], bb[2])
iymax = np.minimum(BBGT[:, 3], bb[3])
iw = np.maximum(ixmax - ixmin + 1., 0.)
ih = np.maximum(iymax - iymin + 1., 0.)
inters = iw * ih
# union
uni = ((bb[2] - bb[0] + 1.) * (bb[3] - bb[1] + 1.) +
(BBGT[:, 2] - BBGT[:, 0] + 1.) *
(BBGT[:, 3] - BBGT[:, 1] + 1.) - inters)
overlaps = inters / uni
ovmax = np.max(overlaps)
jmax = np.argmax(overlaps)
if ovmax > ovthresh:
if not R['difficult'][jmax]:
if not R['det'][jmax]:
tp[d] = 1.
R['det'][jmax] = 1
else:
fp[d] = 1.
else:
fp[d] = 1.
# compute precision recall
fp = np.cumsum(fp)
tp = np.cumsum(tp)
rec = tp / float(npos)
# avoid divide by zero in case the first detection matches a difficult
# ground truth
prec = tp / np.maximum(tp + fp, np.finfo(np.float64).eps)
ap = voc_ap(rec, prec, use_07_metric)
return rec, prec, ap

View File

@@ -0,0 +1,59 @@
import xml.etree.ElementTree as ET
import pickle
import os
from os import listdir, getcwd
from os.path import join
sets=[('2012', 'train'), ('2012', 'val'), ('2007', 'train'), ('2007', 'val'), ('2007', 'test')]
classes = ["aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
def convert(size, box):
dw = 1./(size[0])
dh = 1./(size[1])
x = (box[0] + box[1])/2.0 - 1
y = (box[2] + box[3])/2.0 - 1
w = box[1] - box[0]
h = box[3] - box[2]
x = x*dw
w = w*dw
y = y*dh
h = h*dh
return (x,y,w,h)
def convert_annotation(year, image_id):
in_file = open('VOCdevkit/VOC%s/Annotations/%s.xml'%(year, image_id))
out_file = open('VOCdevkit/VOC%s/labels/%s.txt'%(year, image_id), 'w')
tree=ET.parse(in_file)
root = tree.getroot()
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)
for obj in root.iter('object'):
difficult = obj.find('difficult').text
cls = obj.find('name').text
if cls not in classes or int(difficult)==1:
continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))
bb = convert((w,h), b)
out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
wd = getcwd()
for year, image_set in sets:
if not os.path.exists('VOCdevkit/VOC%s/labels/'%(year)):
os.makedirs('VOCdevkit/VOC%s/labels/'%(year))
image_ids = open('VOCdevkit/VOC%s/ImageSets/Main/%s.txt'%(year, image_set)).read().strip().split()
list_file = open('%s_%s.txt'%(year, image_set), 'w')
for image_id in image_ids:
list_file.write('%s/VOCdevkit/VOC%s/JPEGImages/%s.jpg\n'%(wd, year, image_id))
convert_annotation(year, image_id)
list_file.close()
os.system("cat 2007_train.txt 2007_val.txt 2012_train.txt 2012_val.txt > train.txt")
os.system("cat 2007_train.txt 2007_val.txt 2007_test.txt 2012_train.txt 2012_val.txt > train.all.txt")

View File

@@ -0,0 +1,56 @@
import xml.etree.ElementTree as ET
import pickle
import os
from os import listdir, getcwd
from os.path import join
sets=[('2012', 'val'),('2007', 'test')]
classes = ["aeroplane", "bicycle", "bird", "boat", "bottle", "bus", "car", "cat", "chair", "cow", "diningtable", "dog", "horse", "motorbike", "person", "pottedplant", "sheep", "sofa", "train", "tvmonitor"]
def convert(size, box):
dw = 1./(size[0])
dh = 1./(size[1])
x = (box[0] + box[1])/2.0 - 1
y = (box[2] + box[3])/2.0 - 1
w = box[1] - box[0]
h = box[3] - box[2]
x = x*dw
w = w*dw
y = y*dh
h = h*dh
return (x,y,w,h)
def convert_annotation(year, image_id):
in_file = open('VOCdevkit/VOC%s/Annotations/%s.xml'%(year, image_id))
out_file = open('VOCdevkit/VOC%s/labels/difficult_%s.txt'%(year, image_id), 'w')
tree=ET.parse(in_file)
root = tree.getroot()
size = root.find('size')
w = int(size.find('width').text)
h = int(size.find('height').text)
for obj in root.iter('object'):
difficult = obj.find('difficult').text
cls = obj.find('name').text
if cls not in classes or int(difficult) == 0:
continue
cls_id = classes.index(cls)
xmlbox = obj.find('bndbox')
b = (float(xmlbox.find('xmin').text), float(xmlbox.find('xmax').text), float(xmlbox.find('ymin').text), float(xmlbox.find('ymax').text))
bb = convert((w,h), b)
out_file.write(str(cls_id) + " " + " ".join([str(a) for a in bb]) + '\n')
wd = getcwd()
for year, image_set in sets:
if not os.path.exists('VOCdevkit/VOC%s/labels/'%(year)):
os.makedirs('VOCdevkit/VOC%s/labels/'%(year))
image_ids = open('VOCdevkit/VOC%s/ImageSets/Main/%s.txt'%(year, image_set)).read().strip().split()
list_file = open('difficult_%s_%s.txt'%(year, image_set), 'w')
for image_id in image_ids:
list_file.write('%s/VOCdevkit/VOC%s/JPEGImages/difficult_%s.jpg\n'%(wd, year, image_id))
convert_annotation(year, image_id)
list_file.close()

View File

@@ -0,0 +1,50 @@
#!/bin/bash
dataset=$1
w=$2
h=$3
# Parameters: Human3, CarScale, Human6, Biker
#w=480
#h=640
IFS=','
export LC_NUMERIC="en_US.UTF-8"
wd=`pwd`
dataset_path="data/$dataset"
class_id=0
num=1
mkdir data
wget http://cvlab.hanyang.ac.kr/tracker_benchmark/seq/$dataset.zip
unzip -o $dataset.zip -d data
sed -i.bak 's/\o11/,/g' $dataset_path/groundtruth_rect.txt
sed -i.bak 's/\o11/,/g' $dataset_path/groundtruth_rect.txt
dos2unix $dataset_path/groundtruth_rect.txt
while read -r left right width height; do
filename=$(printf "$dataset_path/img/%04d.txt" $num)
#rm $filename.txt
echo "$class_id " > $filename
printf "%.5f " "$(($((left + width/2)) * 100000 / $w))e-5" >> $filename
printf "%.5f " "$(($((right + height/2)) * 100000 / $h))e-5" >> $filename
printf "%.5f " "$(($((width)) * 100000 / $w))e-5" >> $filename
printf "%.5f " "$(($((height)) * 100000 / $h))e-5" >> $filename
num=$((num + 1))
done < $dataset_path/groundtruth_rect.txt
echo "$dataset" > $dataset_path/otb.names
find $dataset_path/img -name \*.jpg > data/$dataset/train.txt
echo "classes = 1" > data/otb_$dataset.data
echo "train = data/$dataset/train.txt" >> data/otb_$dataset.data
echo "valid = data/$dataset/train.txt" >> data/otb_$dataset.data
echo "names = $dataset_path/otb.names" >> data/otb_$dataset.data
echo "backup = backup/" >> data/otb_$dataset.data
echo "results= results/" >> data/otb_$dataset.data

View File

@@ -0,0 +1,19 @@
echo Run install_cygwin.cmd before:
c:\cygwin64\bin\wget https://pjreddie.com/media/files/cifar.tgz
c:\cygwin64\bin\gzip -d "%CD:\=/%/cifar.tgz"
c:\cygwin64\bin\tar --force-local -xvf "%CD:\=/%/cifar.tar"
c:\cygwin64\bin\cat "%CD:\=/%/labels.txt"
c:\cygwin64\bin\find "%CD:\=/%/cifar/train" -name \*.png > "%CD:\=/%/cifar/train.list"
c:\cygwin64\bin\find "%CD:\=/%/cifar/test" -name \*.png > "%CD:\=/%/cifar/test.list"
pause

View File

@@ -0,0 +1,25 @@
echo Run install_cygwin.cmd before:
rem http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads
rem https://github.com/amd/OpenCL-caffe/wiki/Instructions-to-create-ImageNet-2012-data
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; echo $PWD"
echo Wait several hours...
c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar
c:\cygwin64\bin\mkdir -p "%CD:\=/%/ILSVRC2012_img_train"
c:\cygwin64\bin\tar --force-local -xf "%CD:\=/%/ILSVRC2012_img_train.tar" -C "%CD:\=/%/ILSVRC2012_img_train"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/windows_imagenet_train.sh"
c:\cygwin64\bin\find "%CD:\=/%/ILSVRC2012_img_train" -name \*.JPEG > imagenet1k.train.list
pause

View File

@@ -0,0 +1,36 @@
echo Run install_cygwin.cmd before:
rem http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads
rem https://github.com/amd/OpenCL-caffe/wiki/Instructions-to-create-ImageNet-2012-data
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; echo $PWD"
c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_bbox_val_v3.tgz
c:\cygwin64\bin\gzip -d "%CD:\=/%/ILSVRC2012_bbox_val_v3.tgz"
c:\cygwin64\bin\tar --force-local -xvf "%CD:\=/%/ILSVRC2012_bbox_val_v3.tar"
c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_val.tar
c:\cygwin64\bin\mkdir -p "%CD:\=/%/imgs"
c:\cygwin64\bin\tar --force-local -xf "%CD:\=/%/ILSVRC2012_img_val.tar" -C "%CD:\=/%/imgs"
echo Wait a few hours...
rem c:\cygwin64\bin\wget https://pjreddie.com/media/files/imagenet_label.sh
c:\cygwin64\bin\dos2unix "%CD:\=/%/windows_imagenet_label.sh"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/windows_imagenet_label.sh"
c:\cygwin64\bin\find "%CD:\=/%/labelled" -name \*.JPEG > inet.val.list
pause

View File

@@ -0,0 +1,18 @@
echo Run install_cygwin.cmd before:
rem http://cvlab.hanyang.ac.kr/tracker_benchmark/datasets.html
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; echo $PWD"
c:\cygwin64\bin\dos2unix "%CD:\=/%/otb_get_labels.sh"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/otb_get_labels.sh Suv 320 240"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/otb_get_labels.sh Liquor 640 480"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/otb_get_labels.sh Freeman4 360 240"
c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/otb_get_labels.sh Human3 480 640"
pause

View File

@@ -0,0 +1,16 @@
#!/bin/bash
mkdir -p labelled
wd=`pwd`
for f in val/*.xml;
do
label=`grep -m1 "<name>" $f | grep -oP '<name>\K[^<]*'`
im=`echo $f | sed 's/val/imgs/; s/xml/JPEG/'`
out=`echo $im | sed 's/JPEG/'${label}'.JPEG/; s/imgs/labelled/'`
mv ${wd}/$im ${wd}/$out
#ln -s ${wd}/$im ${wd}/$out
done
#find ${wd}/labelled -name \*.JPEG > inet.val.list

View File

@@ -0,0 +1,17 @@
#!/bin/bash
#wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar
#mkdir -p ILSVRC2012_img_train
#tar --force-local -xf ILSVRC2012_img_train.tar -C ILSVRC2012_img_train
wd=`pwd`
for f in ILSVRC2012_img_train/*.tar;
do
name=$(echo "$f" | cut -f 1 -d '.')
mkdir "${wd}/${name}"
tar --force-local -xf "${wd}/${f}" -C "${wd}/${name}"
done
#find "${wd}/ILSVRC2012_img_train" -name \*.JPEG > imagenet1k.train.list