robocar-road/road/detect.go

package road

import (
	"github.com/cyrilix/robocar-protobuf/go/events"
	"go.uber.org/zap"
	"gocv.io/x/gocv"
	"image"
	"image/color"
	"math"
)

type DetectorOption func(*Detector)

func WithWhiteFilter(lowThreshold int, highThreshold int) DetectorOption {
	return func(d *Detector) {
		err := d.lowerWhite.Close()
		if err != nil {
			zap.S().Errorf("unable to close lower white filter: %v", err)
		}
		err = d.upperWhite.Close()
		if err != nil {
			zap.S().Errorf("unable to close upper white filter: %v", err)
		}

		d.lowerWhite = gocv.NewMatFromScalar(gocv.Scalar{Val1: float64(lowThreshold), Val2: float64(lowThreshold), Val3: float64(lowThreshold), Val4: float64(lowThreshold)}, gocv.MatTypeCV8U)
		d.upperWhite = gocv.NewMatFromScalar(gocv.Scalar{Val1: float64(highThreshold), Val2: float64(highThreshold), Val3: float64(highThreshold), Val4: float64(highThreshold)}, gocv.MatTypeCV8U)
	}
}

func WithYellowFilter(lower gocv.Mat, upper gocv.Mat) DetectorOption {
	return func(d *Detector) {
		err := d.lowerYellow.Close()
		if err != nil {
			zap.S().Errorf("unable to close lower yellow filter: %v", err)
		}
		err = d.upperYellow.Close()
		if err != nil {
			zap.S().Errorf("unable to close upper yellow filter: %v", err)
		}

		d.lowerYellow = lower
		d.upperYellow = upper
	}
}

func WithRegionOfInterest(imgWidth int, imgHeight int, horizon int) DetectorOption {
	roi := buildRegionOfInterest(imgWidth, imgHeight, horizon)
	return func(d *Detector) {
		err := d.roiMask.Close()
		if err != nil {
			zap.S().Errorf("unable to close roi mask: %v", err)
		}
		d.roiMask = *roi
	}
}

func WithPointOnRoad(pt image.Point) DetectorOption {
	return func(d *Detector) {
		d.pointOnRoad = pt
	}
}

func WithCanny(lowThreshold int, highThreshold int) DetectorOption {
	return func(d *Detector) {
		d.cannyLowThreshold = float32(lowThreshold)
		d.cannyHighThreshold = float32(highThreshold)
	}
}

func WithGaussianBlur(kernelSize int) DetectorOption {
	return func(d *Detector) {
		d.gaussianBlurKernelSize = kernelSize
	}
}

func WithHoughLines(rho int, theta float32, threshold int, minLineLength int, maxLineGap int) DetectorOption {
	return func(d *Detector) {
		d.houghLinesRho = float32(rho)
		d.houghLinesTheta = theta
		d.houghLinesThreshold = threshold
		d.houghLinesMinLineLength = minLineLength
		d.houghLinesMaxLineGap = maxLineGap
	}
}

func NewDetector(options ...DetectorOption) *Detector {
	whiteThreshold := 20.

	roiMask := buildRegionOfInterest(160, 120, 110)

	pointOnRoad := image.Point{X: 160 / 2, Y: 120 - 30}

	d := Detector{
		lowerWhite: gocv.NewMatFromScalar(gocv.Scalar{Val1: whiteThreshold, Val2: whiteThreshold, Val3: whiteThreshold, Val4: whiteThreshold}, gocv.MatTypeCV8U),
		upperWhite: gocv.NewMatFromScalar(gocv.Scalar{Val1: 255., Val2: 255., Val3: 255., Val4: 255.}, gocv.MatTypeCV8U),

		lowerYellow: gocv.NewMatFromScalar(gocv.Scalar{Val1: 90., Val2: 100., Val3: 100.}, gocv.MatTypeCV8U),
		upperYellow: gocv.NewMatFromScalar(gocv.Scalar{Val1: 110., Val2: 255., Val3: 255.}, gocv.MatTypeCV8U),

		cannyLowThreshold:  100.,
		cannyHighThreshold: 250.,

		gaussianBlurKernelSize: 3,

		// Hough Transform
		houghLinesRho:           2,                 // distance resolution in pixels of the Hough grid
		houghLinesTheta:         1 * math.Pi / 180, // angular resolution in radians of the Hough grid
		houghLinesThreshold:     15,                // minimum number of votes (intersections in Hough grid cell)
		houghLinesMinLineLength: 10,                // minimum number of pixels making up a line
		houghLinesMaxLineGap:    20,                // maximum gap in pixels between connectable line segments

		roiMask: *roiMask,

		pointOnRoad: pointOnRoad,
	}

	for _, option := range options {
		option(&d)
	}
	return &d
}

func buildRegionOfInterest(imgWidth int, imgHeight int, horizon int) *gocv.Mat {
	// RegionOfInterest
	// defining a blank roiMask to start with
	roiMask := gocv.Zeros(imgHeight, imgWidth, gocv.MatTypeCV8U)

	vertices := gocv.NewPointsVectorFromPoints(
		[][]image.Point{
			{
				{0, imgHeight - horizon},
				{imgWidth, imgHeight - horizon},
				{imgWidth, imgHeight},
				{0, imgHeight},
			},
		},
	)
	defer vertices.Close()
	ignoreMaskColor := color.RGBA{R: 255, G: 255, B: 255, A: 255}
	gocv.FillPoly(&roiMask, vertices, ignoreMaskColor)
	return &roiMask
}

type Detector struct {
	whiteThreshold float64
	lowerWhite     gocv.Mat
	upperWhite     gocv.Mat

	lowerYellow gocv.Mat
	upperYellow gocv.Mat

	cannyLowThreshold, cannyHighThreshold float32

	gaussianBlurKernelSize int

	// Hough lines parameters
	houghLinesRho, houghLinesTheta                float32
	houghLinesThreshold                           int
	houghLinesMinLineLength, houghLinesMaxLineGap int

	// Region Of interest
	roiMask gocv.Mat

	pointOnRoad image.Point
}

func (d *Detector) Close() {
	defer func(lowerWhite *gocv.Mat) {
		err := lowerWhite.Close()
		if err != nil {
			zap.S().Errorf("unable to close lowerWhite: %v", err)
		}
	}(&d.lowerWhite)

	defer func(upperWhite *gocv.Mat) {
		err := upperWhite.Close()
		if err != nil {
			zap.S().Errorf("unable to close upperWhite: %v", err)
		}
	}(&d.upperWhite)

	defer func(lowerYellow *gocv.Mat) {
		err := lowerYellow.Close()
		if err != nil {
			zap.S().Errorf("unable to close lowerYellow: %v", err)
		}
	}(&d.lowerYellow)

	defer func(upperYellow *gocv.Mat) {
		err := upperYellow.Close()
		if err != nil {
			zap.S().Errorf("unable to close upperYellow: %v", err)
		}
	}(&d.upperYellow)

	defer func(roiMask *gocv.Mat) {
		err := roiMask.Close()
		if err != nil {
			zap.S().Errorf("unable to close roiMask: %v", err)
		}
	}(&d.roiMask)
}

func (d *Detector) Detect(img *gocv.Mat) ([]*events.Point, *events.Ellipse) {
	// Only keep white and yellow pixels in the image, all other pixels become black
	imgFiltered := d.filterColors(img)
	defer func(imgFiltered *gocv.Mat) {
		err := imgFiltered.Close()
		if err != nil {
			zap.S().Errorf("unable to close imgFiltered: %v", err)
		}
	}(imgFiltered)

	// # Read in and grayscale the image
	imgGray := d.grayscale(imgFiltered)
	defer func(imgGray *gocv.Mat) {
		err := imgGray.Close()
		if err != nil {
			zap.S().Errorf("unable to close imgGray: %v", err)
		}
	}(imgGray)

	// Apply Gaussian smoothing
	blurGray := d.gaussianBlur(imgGray)
	defer func(blurGray *gocv.Mat) {
		err := blurGray.Close()
		if err != nil {
			zap.S().Errorf("unable to close blurGray: %v", err)
		}
	}(blurGray)

	// Apply Canny Edge Detector
	edges := d.canny(blurGray)
	defer func(edges *gocv.Mat) {
		err := edges.Close()
		if err != nil {
			zap.S().Errorf("unable to close edges: %v", err)
		}
	}(edges)

	maskedEdges := d.applyRegionOfInterest(edges)
	defer func(maskedEdges *gocv.Mat) {
		err := maskedEdges.Close()
		if err != nil {
			zap.S().Errorf("unable to close maskedEdges: %v", err)
		}
	}(maskedEdges)

	// Run Hough on edge detected image
	road, ellipsis := d.getRoadShapeWithHoughLines(maskedEdges)
	return road, ellipsis
}

// Filter the image to include only yellow and white pixels
func (d *Detector) filterColors(img *gocv.Mat) *gocv.Mat {

	// Filter white pixels
	whiteMask := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8UC3)
	defer func(whiteMask *gocv.Mat) {
		err := whiteMask.Close()
		if err != nil {
			zap.S().Errorf("unable to close whiteMask: %v", err)
		}
	}(&whiteMask)
	gocv.InRange(*img, d.lowerWhite, d.upperWhite, &whiteMask)
	// Convert one channel result to 3 channel mask
	gocv.Merge([]gocv.Mat{whiteMask, whiteMask, whiteMask}, &whiteMask)

	whiteImage := gocv.NewMatWithSize(img.Rows(), img.Cols(), img.Type())
	defer func(whiteImage *gocv.Mat) {
		err := whiteImage.Close()
		if err != nil {
			zap.S().Errorf("unable to close whiteImage: %v", err)
		}
	}(&whiteImage)
	gocv.BitwiseAnd(*img, whiteMask, &whiteImage)

	// Filter yellow pixels
	hsv := gocv.NewMatWithSize(img.Rows(), img.Cols(), img.Type())
	defer func(hsv *gocv.Mat) {
		err := hsv.Close()
		if err != nil {
			zap.S().Errorf("unable to close hsv: %v", err)
		}
	}(&hsv)
	gocv.CvtColor(*img, &hsv, gocv.ColorBGRToHSV)

	yellowMask := gocv.NewMatWithSize(img.Rows(), img.Cols(), img.Type())
	defer func(yellowMask *gocv.Mat) {
		err := yellowMask.Close()
		if err != nil {
			zap.S().Errorf("unable to close yellowMask: %v", err)
		}
	}(&yellowMask)
	gocv.InRange(hsv, d.lowerYellow, d.upperYellow, &yellowMask)
	// Convert one channel result to 3 channel mask
	gocv.Merge([]gocv.Mat{yellowMask, yellowMask, yellowMask}, &yellowMask)

	yellowImage := gocv.NewMatWithSize(img.Rows(), img.Cols(), img.Type())
	defer func(yellowImage *gocv.Mat) {
		err := yellowImage.Close()
		if err != nil {
			zap.S().Errorf("unable to close yellowImage: %v", err)
		}
	}(&yellowImage)
	gocv.BitwiseAnd(*img, yellowMask, &yellowImage)

	// Combine the two above images
	image2 := gocv.NewMatWithSize(img.Rows(), img.Cols(), img.Type())
	gocv.AddWeighted(whiteImage, 1., yellowImage, 1., 0., &image2)
	return &image2
}

/*
	Applies the Grayscale transform

This will return an image with only one color channel
but NOTE: to see the returned image as grayscale
you should call plt.imshow(gray, cmap='gray')
*/
func (d *Detector) grayscale(img *gocv.Mat) *gocv.Mat {
	grayImg := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8U)
	gocv.CvtColor(*img, &grayImg, gocv.ColorBGRToGray)

	return &grayImg
}

/* Applies the Canny transform */
func (d *Detector) canny(img *gocv.Mat) *gocv.Mat {
	edges := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8U)
	gocv.Canny(*img, &edges, d.cannyLowThreshold, d.cannyHighThreshold)

	return &edges
}

func (d *Detector) gaussianBlur(img *gocv.Mat) *gocv.Mat {
	blur := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8U)
	gocv.GaussianBlur(*img, &blur, image.Point{X: d.gaussianBlurKernelSize, Y: d.gaussianBlurKernelSize}, 0., 0., gocv.BorderDefault)
	return &blur
}

func (d *Detector) getRoadShapeWithHoughLines(img *gocv.Mat) ([]*events.Point, *events.Ellipse) {
	lines := gocv.NewMat()
	defer func(lines *gocv.Mat) {
		err := lines.Close()
		if err != nil {
			zap.S().Errorf("unable to close lines mar: %v", err)
		}
	}(&lines)

	gocv.HoughLinesP(*img, &lines, d.houghLinesRho, d.houghLinesTheta, d.houghLinesThreshold)

	// Generate new image with detected edges
	imgLines := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8U)
	defer func(imgLines *gocv.Mat) {
		err := imgLines.Close()
		if err != nil {
			zap.S().Errorf("unable to close imgLines: %v", err)
		}
	}(&imgLines)

	for i := 0; i < lines.Rows(); i++ {
		x1 := lines.GetIntAt(i, 0)
		y1 := lines.GetIntAt(i, 1)
		x2 := lines.GetIntAt(i, 2)
		y2 := lines.GetIntAt(i, 3)

		gocv.Line(&imgLines, image.Point{X: int(x1), Y: int(y1)}, image.Point{X: int(x2), Y: int(y2)}, color.RGBA{
			R: 255,
			G: 255,
			B: 255,
			A: 255,
		}, 3)
	}

	// Draw rectangle to add limit on image border
	gocv.Rectangle(&imgLines, image.Rectangle{
		Min: image.Point{X: 0, Y: 0},
		Max: image.Point{X: img.Cols() - 1, Y: img.Rows() - 1},
	}, color.RGBA{R: 255, G: 255, B: 255, A: 255}, 1)

	kernel := gocv.Ones(8, 8, gocv.MatTypeCV8U)
	defer func(kernel *gocv.Mat) {
		err := kernel.Close()
		if err != nil {
			zap.S().Errorf("unable to close kernel: %v", err)
		}
	}(&kernel)
	gocv.Dilate(imgLines, &imgLines, kernel)
	gocv.Erode(imgLines, &imgLines, kernel)

	cnts := gocv.FindContours(imgLines, gocv.RetrievalList, gocv.ChainApproxSimple)
	defer cnts.Close()

	for i := 0; i < cnts.Size(); i++ {
		cnt := cnts.At(i)
		pv := gocv.ApproxPolyDP(cnt, 0.01*gocv.ArcLength(cnt, true), true)
		if gocv.PointPolygonTest(pv, d.pointOnRoad, false) > 0 {
			ellipse := d.computeEllipsis(&pv)
			cntr_result := make([]*events.Point, 0, pv.Size())
			for i := 0; i < pv.Size(); i++ {
				pt := pv.At(i)
				cntr_result = append(cntr_result, &events.Point{X: int32(pt.X), Y: int32(pt.Y)})
			}
			return cntr_result, ellipse
		}
	}
	return []*events.Point{}, &EllipseNotFound
}

/*
Applies an image mask.

	Only keeps the region of the image defined by the polygon
	formed from `vertices`. The rest of the image is set to black.
*/
func (d *Detector) applyRegionOfInterest(img *gocv.Mat) *gocv.Mat {

	// returning the image only where mask pixels are nonzero
	maskedImage := gocv.NewMatWithSize(img.Rows(), img.Cols(), gocv.MatTypeCV8U)

	gocv.BitwiseAnd(*img, d.roiMask, &maskedImage)
	return &maskedImage
}

var EllipseNotFound = events.Ellipse{Confidence: 0.}

func (d *Detector) computeEllipsis(road *gocv.PointVector) *events.Ellipse {
	if road.Size() < 5 {
		return &EllipseNotFound
	}

	rotatedRect := gocv.FitEllipse(*road)

	trust := d.computeTrustFromCenter(&rotatedRect.Center)
	zap.S().Debugf("Trust: %v", trust)

	return &events.Ellipse{
		Center: &events.Point{
			X: int32(rotatedRect.Center.X),
			Y: int32(rotatedRect.Center.Y),
		},
		Width:      int32(rotatedRect.Width),
		Height:     int32(rotatedRect.Height),
		Angle:      float32(rotatedRect.Angle),
		Confidence: d.computeTrustFromCenter(&rotatedRect.Center),
	}
}

func (d *Detector) computeTrustFromCenter(ellipsisCenter *image.Point) float32 {
	safeMinX := 48
	safeMaxX := 115
	safeMinY := 69
	safeMaxY := 119

	if safeMinX <= ellipsisCenter.X && ellipsisCenter.X <= safeMaxX && safeMinY <= ellipsisCenter.Y && ellipsisCenter.Y <= safeMaxY {
		return 1.0
	}

	if safeMinX <= ellipsisCenter.X && ellipsisCenter.X <= safeMaxX {
		return d.computeTrustOnAxis(safeMaxY, safeMinY, ellipsisCenter.Y)
	}

	if safeMinY <= ellipsisCenter.Y && ellipsisCenter.Y <= safeMaxY {
		return d.computeTrustOnAxis(safeMaxX, safeMinX, ellipsisCenter.X)
	}

	return d.computeTrustOnAxis(safeMaxY, safeMinY, ellipsisCenter.Y) * d.computeTrustOnAxis(safeMaxX, safeMinX, ellipsisCenter.X)
}

func (d *Detector) computeTrustOnAxis(safeMax, safeMin, value int) float32 {
	trust := 1.
	if value > safeMax {
		trust = 1. / float64(value-safeMax)
	} else if value < safeMin {
		trust = 1. / float64(safeMin-value)
	}
	trust = trust * 10.
	if trust > 0.9 {
		trust = 0.9
	}
	if trust < 0. {
		trust = 0.
	}
	return float32(trust)

}