feat(dkimport): add command to import data from donkey parts

This commit is contained in:
Cyrille Nofficial 2021-10-17 19:16:37 +02:00
parent 538cea18f2
commit 59b4477fb5
31 changed files with 267 additions and 0 deletions

166
dkimpt/import.go Normal file
View File

@ -0,0 +1,166 @@
package dkimpt
import (
"encoding/json"
"fmt"
"github.com/cyrilix/robocar-tools/record"
log "github.com/sirupsen/logrus"
"io/ioutil"
"os"
"path"
"regexp"
)
/* donkey import*/
var (
camSubDir = "cam"
camIndexRegexp *regexp.Regexp
recordIndexRegexp *regexp.Regexp
)
func ImportDonkeyRecords(basedir string, destDir string) error {
dirItems, err := ioutil.ReadDir(basedir)
if err != nil {
return fmt.Errorf("unable to list directory in %v dir: %v", basedir, err)
}
imgCams := make([]string, 0)
records := make([]string, 0)
for _, dirItem := range dirItems {
log.Debugf("process %v directory", dirItem)
camDir := path.Join(destDir, dirItem.Name(), camSubDir)
err := os.MkdirAll(camDir, os.FileMode(0755))
if err != nil {
return fmt.Errorf("unable to make dest directories %v: %v", camDir, err)
}
imgDir := path.Join(basedir, dirItem.Name(), camSubDir)
imgs, err := ioutil.ReadDir(imgDir)
if err != nil {
return fmt.Errorf("unable to list cam images in directory %v: %v", imgDir, err)
}
for _, img := range imgs {
idx, err := indexFromFile(camIndexRegexp, img.Name())
if err != nil {
return fmt.Errorf("unable to find index in cam image name %v: %v", img.Name(), err)
}
log.Debugf("found image with index %v", idx)
records = append(records, path.Join(basedir, dirItem.Name(), fmt.Sprintf(record.RecorNameFormat, idx)))
imgCams = append(imgCams, path.Join(basedir, dirItem.Name(), camSubDir, img.Name()))
}
err = copyToDestdir(destDir, dirItem.Name(), &imgCams, &records)
if err != nil {
log.Warnf("unable to copy files from %v to %v: %v", path.Join(basedir, dirItem.Name()), destDir, err)
continue
}
}
return nil
}
func init() {
re, err := regexp.Compile("image_array_(?P<idx>[0-9]+)\\.jpg$")
if err != nil {
log.Fatalf("unable to compile regex: %v", err)
}
camIndexRegexp = re
re, err = regexp.Compile("record_(?P<idx>[0-9]+)\\.json$")
if err != nil {
log.Fatalf("unable to compile regex: %v", err)
}
recordIndexRegexp = re
}
func indexFromFile(regex *regexp.Regexp, fileName string) (string, error) {
matches := findNamedMatches(regex, fileName)
if matches["idx"] == "" {
return "", fmt.Errorf("no index in filename")
}
return matches["idx"], nil
}
func findNamedMatches(regex *regexp.Regexp, str string) map[string]string {
match := regex.FindStringSubmatch(str)
results := map[string]string{}
for i, name := range match {
results[regex.SubexpNames()[i]] = name
}
return results
}
func copyToDestdir(destdir, dirItem string, imgFiles *[]string, recordFiles *[]string) error {
err := copyJsonFiles(destdir, dirItem, recordFiles)
if err != nil {
return fmt.Errorf("unable to copy json files in %v directory: %v", destdir, err)
}
err = copyCamImages(destdir, dirItem, imgFiles)
if err != nil {
return fmt.Errorf("unable to copy cam files in %v director: %v", destdir, err)
}
return nil
}
func copyCamImages(destdir, dirItem string, imgFiles *[]string) error {
for _, img := range *imgFiles {
imgContent, err := ioutil.ReadFile(img)
if err != nil {
return fmt.Errorf("unable to read img: %v", err)
}
idx, err := indexFromFile(camIndexRegexp, img)
if err != nil {
log.Warningf("unable to extract idx from filename %v: %v", img, err)
continue
}
imgName := path.Join(destdir, dirItem, camSubDir, fmt.Sprintf("image_array_%s_%s.jpg", dirItem, idx))
err = ioutil.WriteFile(imgName, imgContent, os.FileMode(0755))
if err != nil {
return fmt.Errorf("unable to write image %v: %v", imgName, err)
}
}
return nil
}
func copyJsonFiles(destdir, dirItem string, recordFiles *[]string) error {
for _, r := range *recordFiles {
content, err := ioutil.ReadFile(r)
if err != nil {
return fmt.Errorf("unable to read json content: %v", err)
}
idx, err := indexFromFile(recordIndexRegexp, r)
if err != nil {
log.Warningf("unable to extract idx from filename %v: %v", r, err)
continue
}
var rcd record.Record
err = json.Unmarshal(content, &rcd)
if err != nil {
return fmt.Errorf("unable to unmarshal record: %v", err)
}
camName := fmt.Sprintf("image_array_%s_%s.jpg", dirItem, idx)
rcd.CamImageArray = path.Join(camSubDir, camName)
recordBytes, err := json.Marshal(&rcd)
if err != nil {
return fmt.Errorf("unable to marshal %v record: %v", rcd, err)
}
recordFileName := path.Join(destdir, dirItem, fmt.Sprintf("record_%s_%s.json", dirItem, idx))
err = ioutil.WriteFile(recordFileName, recordBytes, os.FileMode(0755))
if err != nil {
return fmt.Errorf("unable to write json record %v: %v", recordFileName, err)
}
}
return nil
}

87
dkimpt/import_test.go Normal file
View File

@ -0,0 +1,87 @@
package dkimpt
import (
"encoding/json"
record2 "github.com/cyrilix/robocar-tools/record"
log "github.com/sirupsen/logrus"
"io/ioutil"
"os"
"path"
"testing"
)
func TestImportDonkeyRecords(t *testing.T) {
destDir, err := ioutil.TempDir("", "test-import")
if err != nil {
t.Errorf("unable to generate destdir for test")
}
defer func() {
err := os.RemoveAll(destDir)
log.Errorf("unable to delete tmpdir %v: %v", destDir, err)
}()
err = ImportDonkeyRecords("testdata", destDir)
if err != nil {
t.Errorf("unable to import files: %v", err)
}
cases := []struct {
setDir string
expectedDir int
expectedRecords int
}{
{"20191012_111416", 1, 7},
{"20191012_122633", 1, 8},
}
for _, c := range cases {
files, err := ioutil.ReadDir(path.Join(destDir, c.setDir))
if err != nil {
t.Errorf("[%v] unable to list %v content: %v", c.setDir, destDir, err)
}
dirNames := make([]string, 0)
fileNames := make([]string, 0)
for _, f := range files {
if f.IsDir() {
dirNames = append(dirNames, f.Name())
} else {
fileNames = append(fileNames, f.Name())
}
}
if len(dirNames) != c.expectedDir {
t.Errorf("[%v] %v dirs found, wants %v", c.setDir, len(dirNames), 1)
}
if len(fileNames) != c.expectedRecords {
t.Errorf("[%v] %v files found, wants %v", c.setDir, len(fileNames), 15)
}
for _, jsonFile := range fileNames {
content, err := ioutil.ReadFile(path.Join(destDir, c.setDir, jsonFile))
if err != nil {
t.Errorf("[%v] unable to read file content of %v: %v", c.setDir, jsonFile, err)
continue
}
var record record2.Record
err = json.Unmarshal(content, &record)
if err != nil {
t.Errorf("[%v] unable to unmarshal file %v with content %v: %v", c.setDir, jsonFile, content, err)
continue
}
if record.UserAngle == 0. {
t.Errorf("[%v] user_angle field not initialized for %v file: %v", c.setDir, jsonFile, content)
}
camPath := path.Join(destDir, c.setDir, record.CamImageArray)
camFile, err := os.Stat(camPath)
if os.IsNotExist(err) {
t.Errorf("[%v] cam image %v doesn't exist for record %v: %v", c.setDir, camPath, jsonFile, err)
continue
}
if camFile.Size() == 0 {
t.Errorf("[%v] cam image %v is empty", c.setDir, camFile.Name())
}
}
}
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 3.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.9 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.6 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.41764705882352937, "cfg/throttle/max": 0.9, "angle": -0.41764705882352937, "sleep_time": 0.029084730148315432, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 0.4830229017469618, "user/throttle": 0.5926680244399185, "rate_htz": 20, "throttle": 0.5926680244399185, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_0_.jpg", "captor/distance": -1, "pilot/throttle": 0.9155570387029685, "img/road_ellipse": "img-road_ellipse_0_.jpg", "road/contour": [[0, 84], [6, 89], [0, 99], [159, 99], [152, 82], [159, 80], [159, 60], [149, 41], [141, 41], [143, 51], [135, 41], [31, 41], [28, 47], [16, 41], [10, 52], [20, 53], [9, 87]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[80, 86], [133.9037322998047, 157.2823944091797], 81.30558776855469, 1.0]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.49607843137254903, "cfg/throttle/max": 0.9, "angle": -0.49607843137254903, "sleep_time": 0.018878984451293948, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 0.12555482652452257, "user/throttle": 0.6069246435845215, "rate_htz": 20, "throttle": 0.6069246435845215, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_1_.jpg", "captor/distance": -1, "pilot/throttle": 0.592365046012554, "img/road_ellipse": "img-road_ellipse_1_.jpg", "road/contour": [[0, 59], [15, 66], [0, 99], [159, 99], [159, 60], [149, 41], [142, 41], [143, 50], [127, 41], [121, 57], [118, 41], [29, 41], [20, 61], [15, 55], [22, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[80, 87], [117.75352478027344, 166.38485717773438], 87.7400131225586, 1.0]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.5058823529411764, "cfg/throttle/max": 0.9, "angle": -0.5058823529411764, "sleep_time": 0.03089265823364258, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 1.0, "user/throttle": 0.6109979633401221, "rate_htz": 20, "throttle": 0.6109979633401221, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_2_.jpg", "captor/distance": -1, "pilot/throttle": 1.0, "img/road_ellipse": "img-road_ellipse_2_.jpg", "road/contour": [[0, 60], [0, 76], [8, 83], [0, 99], [159, 99], [154, 48], [159, 41], [16, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[81, 78], [151.7767791748047, 165.07664489746094], 64.75083923339844, 1.0]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.615686274509804, "cfg/throttle/max": 0.9, "angle": -0.615686274509804, "sleep_time": 0.02732782363891602, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 0.5554974873860676, "user/throttle": 0.6578411405295315, "rate_htz": 20, "throttle": 0.6578411405295315, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_3_.jpg", "captor/distance": -1, "pilot/throttle": 0.885990863031913, "img/road_ellipse": "img-road_ellipse_3_.jpg", "road/contour": [[0, 47], [0, 53], [13, 48], [0, 67], [0, 99], [159, 99], [159, 41], [8, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[81, 74], [142.79656982421875, 170.35728454589844], 80.00104522705078, 1.0]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.792156862745098, "cfg/throttle/max": 0.9, "angle": -0.792156862745098, "sleep_time": 0.033392477035522464, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 0.7277487436930339, "user/throttle": 0.6843177189409368, "rate_htz": 20, "throttle": 0.6843177189409368, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_4_.jpg", "captor/distance": -1, "pilot/throttle": 0.6020408128165022, "img/road_ellipse": "img-road_ellipse_4_.jpg", "road/contour": [[0, 41], [0, 65], [11, 64], [0, 80], [0, 99], [159, 99], [159, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[79, 67], [153.01275634765625, 205.82652282714844], 0.48624387383461, 0.9]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.8274509803921568, "cfg/throttle/max": 0.9, "angle": -0.8274509803921568, "sleep_time": 0.01509957313537598, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 1.0, "user/throttle": 0.6863543788187372, "rate_htz": 20, "throttle": 0.6863543788187372, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_5_.jpg", "captor/distance": -1, "pilot/throttle": 1.0, "img/road_ellipse": "img-road_ellipse_5_.jpg", "road/contour": [[0, 41], [0, 87], [9, 85], [0, 99], [159, 99], [159, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [[79, 70], [147.05279541015625, 147.92002868652344], 0.0, 1.0]}

View File

@ -0,0 +1 @@
{"ctrl/record": true, "cfg/throttle/min": 0.52, "user/angle": -0.8098039215686275, "cfg/throttle/max": 0.9, "angle": -0.8098039215686275, "sleep_time": 0.005940723419189456, "cfg/throttle/angle/dangerous": 0.8, "pilot/angle": 1.0, "user/throttle": 0.6863543788187372, "rate_htz": 20, "throttle": 0.6863543788187372, "cfg/throttle/compute_from_steering": true, "cam/image_array": "cam-image_array_6_.jpg", "captor/distance": -1, "pilot/throttle": 0.52, "img/road_ellipse": "img-road_ellipse_6_.jpg", "road/contour": [[0, 41], [0, 99], [159, 99], [159, 41]], "recording": [0.0, 0.0, "user", false], "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "road/ellipse": [null, null, 90.0, 0.0]}

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 6.2 KiB

View File

@ -0,0 +1 @@
{"road/ellipse": [[93, 65], [86.32437133789062, 127.01263427734375], 84.16026306152344, 0.9], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, null, false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": 0.0, "captor/distance": 3191, "cam/image_array": "cam-image_array_0_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[18, 65], [24, 74], [42, 64], [50, 80], [31, 77], [45, 87], [55, 82], [47, 94], [38, 85], [35, 99], [137, 99], [142, 89], [133, 80], [159, 72], [156, 58], [141, 60], [146, 42], [157, 47], [152, 41], [43, 41], [52, 54]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_0_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[78, 71], [99.24079895019531, 158.44842529296875], 83.14945983886719, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.7877869129180908, "captor/distance": 3220, "cam/image_array": "cam-image_array_1_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [159, 95], [143, 94], [138, 75], [110, 85], [104, 76], [128, 64], [159, 73], [156, 58], [141, 60], [154, 41], [43, 41], [53, 53], [33, 64], [28, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_1_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[80, 77], [110.85153198242188, 157.39715576171875], 77.76525115966797, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.0079838752746582, "captor/distance": 3237, "cam/image_array": "cam-image_array_2_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [159, 95], [149, 95], [152, 87], [141, 95], [133, 84], [140, 72], [159, 73], [156, 58], [141, 60], [146, 42], [158, 47], [152, 41], [43, 41], [53, 53], [39, 53], [33, 64], [22, 51], [28, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_2_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[79, 83], [127.54984283447266, 160.6329803466797], 73.02867126464844, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.08004755973815918, "captor/distance": 3175, "cam/image_array": "cam-image_array_3_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [156, 96], [133, 84], [140, 72], [159, 73], [156, 58], [141, 60], [146, 42], [158, 47], [154, 41], [43, 41], [53, 53], [33, 64], [22, 51], [28, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_3_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[77, 96], [145.10252380371094, 171.11947631835938], 58.72574996948242, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.061002683639526364, "captor/distance": 2359, "cam/image_array": "cam-image_array_4_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [156, 96], [133, 84], [140, 72], [159, 68], [156, 58], [141, 60], [146, 42], [157, 47], [152, 41], [43, 41], [53, 53], [39, 53], [33, 64], [22, 50], [28, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_4_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[79, 83], [126.231689453125, 159.87911987304688], 74.92227172851562, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.07351298332214355, "captor/distance": 3197, "cam/image_array": "cam-image_array_5_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [156, 96], [133, 84], [140, 72], [159, 73], [156, 58], [141, 60], [146, 42], [157, 47], [152, 41], [43, 41], [53, 53], [33, 64], [22, 51], [28, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_5_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}

View File

@ -0,0 +1 @@
{"road/ellipse": [[80, 77], [119.39763641357422, 156.11976623535156], 73.90866088867188, 1.0], "cfg/throttle/min": 0.4, "pilot/angle": -0.2857142857142858, "cfg/throttle/max": 0.6, "recording": [0.0, 0.0, "user", false], "user/angle": -1.0, "cfg/throttle/compute_from_steering": true, "angle": -1.0, "rate_htz": 20, "sleep_time": -0.06748814582824707, "captor/distance": 3196, "cam/image_array": "cam-image_array_6_.jpg", "ctrl/record": true, "throttle": 1.0, "cfg/throttle/angle/dangerous": 0.8, "road/contour": [[0, 41], [0, 99], [159, 95], [149, 95], [152, 87], [141, 95], [133, 84], [140, 72], [159, 73], [156, 58], [141, 60], [146, 42], [158, 47], [154, 41], [43, 41], [53, 53], [33, 64], [24, 58], [25, 41]], "pilot/throttle": 0.4, "img/road_ellipse": "img-road_ellipse_6_.jpg", "user/mode": "user", "cfg/throttle/angle/safe": 0.3, "user/throttle": 1.0}