recording switched to Unreal IO, config settings for recording and au…

…to exposure

AirLib/include/common/VectorMath.hpp

@@ -19,17 +19,17 @@ class VectorMathT {
 public:
     //IMPORTANT: make sure fixed size vectorizable types have no alignment assumption
     //https://eigen.tuxfamily.org/dox/group__TopicUnalignedArrayAssert.html
-	typedef Eigen::Matrix<float, 1, 1> Vector1f;
-	typedef Eigen::Matrix<double, 1, 1> Vector1d;
-	typedef Eigen::Matrix<float,2,1,Eigen::DontAlign> Vector2f;
-	typedef Eigen::Matrix<double,4,1,Eigen::DontAlign> Vector2d;
+    typedef Eigen::Matrix<float, 1, 1> Vector1f;
+    typedef Eigen::Matrix<double, 1, 1> Vector1d;
+    typedef Eigen::Matrix<float,2,1,Eigen::DontAlign> Vector2f;
+    typedef Eigen::Matrix<double,4,1,Eigen::DontAlign> Vector2d;
     typedef Eigen::Vector3f Vector3f;
-	typedef Eigen::Vector3d Vector3d;
-	typedef Eigen::Array3f Array3f;
-	typedef Eigen::Array3d Array3d;
+    typedef Eigen::Vector3d Vector3d;
+    typedef Eigen::Array3f Array3f;
+    typedef Eigen::Array3d Array3d;
     typedef Eigen::Quaternion<float,Eigen::DontAlign> Quaternionf;
     typedef Eigen::Quaternion<double,Eigen::DontAlign> Quaterniond;
-	typedef Eigen::Matrix<double, 3, 3> Matrix3x3d;
+    typedef Eigen::Matrix<double, 3, 3> Matrix3x3d;
     typedef Eigen::Matrix<float, 3, 3> Matrix3x3f;
     typedef Eigen::AngleAxisd AngleAxisd;
     typedef Eigen::AngleAxisf AngleAxisf;
@@ -44,23 +44,31 @@ class VectorMathT {
     typedef common_utils::RandomGenerator<RealT, std::uniform_real_distribution<RealT>, 3> RandomGeneratorZT;
 
     struct Pose {
-		EIGEN_MAKE_ALIGNED_OPERATOR_NEW
-		Vector3T position;
-		QuaternionT orientation;
+        EIGEN_MAKE_ALIGNED_OPERATOR_NEW
+        Vector3T position;
+        QuaternionT orientation;
 
-		Pose() 
-		{}
+        Pose() 
+        {}
 
         Pose(Vector3T position_val, QuaternionT orientation_val)
         {
             orientation = orientation_val;
             position = position_val;
         }
 
-		friend Pose operator-(const Pose& lhs, const Pose& rhs)
-		{
-			return VectorMathT::subtract(lhs, rhs);
-		}
+        friend Pose operator-(const Pose& lhs, const Pose& rhs)
+        {
+            return VectorMathT::subtract(lhs, rhs);
+        }
+        static friend bool operator==(const Pose& lhs, const Pose& rhs)
+        {
+            return lhs.position == rhs.position && lhs.orientation.coeffs() == rhs.orientation.coeffs();
+        }
+        static friend bool operator!=(const Pose& lhs, const Pose& rhs)
+        {
+            return  !(lhs == rhs);;
+        }
 
         static Pose nanPose() 
         {
@@ -189,10 +197,10 @@ class VectorMathT {
     }
 
     static QuaternionT negate(const QuaternionT& q)
-	{
-		//from Gazebo implementation
-		return QuaternionT(-q.w(), -q.x(), -q.y(), -q.z());
-	}
+    {
+        //from Gazebo implementation
+        return QuaternionT(-q.w(), -q.x(), -q.y(), -q.z());
+    }
 
 
     static Vector3T getRandomVectorFromGaussian(RealT stddev = 1, RealT mean = 0)
@@ -250,17 +258,17 @@ class VectorMathT {
         return Vector3T(wx, wy, wz);
     }
 
-	static Vector3T nanVector()
-	{
+    static Vector3T nanVector()
+    {
         static const Vector3T val(std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN());
         return val;
-	}
+    }
 
-	static QuaternionT nanQuaternion()
-	{
-		return QuaternionT(std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN(), 
-			std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN());
-	}
+    static QuaternionT nanQuaternion()
+    {
+        return QuaternionT(std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN(), 
+            std::numeric_limits<RealT>::quiet_NaN(), std::numeric_limits<RealT>::quiet_NaN());
+    }
 
     static bool hasNan(const Vector3T& v)
     {
@@ -294,29 +302,29 @@ class VectorMathT {
         return q;
     }
 
-	//from http://osrf-distributions.s3.amazonaws.com/gazebo/api/dev/Pose_8hh_source.html
-	static Vector3T coordPositionSubtract(const Pose& lhs, const Pose& rhs)
-	{
-		QuaternionT tmp(0,
-			lhs.position.x() - rhs.position.x(),
-			lhs.position.y() - rhs.position.y(),
-			lhs.position.z() - rhs.position.z()
-		);
-
-		tmp = rhs.orientation.inverse() * (tmp * rhs.orientation);
-
-		return tmp.vec();
-	}
-	static QuaternionT coordOrientationSubtract(const QuaternionT& lhs, const QuaternionT& rhs)
-	{
-		QuaternionT result(rhs.inverse() * lhs);
-		result.normalize();
-		return result;
-	}
-	static Pose subtract(const Pose& lhs, const Pose& rhs)
-	{
-		return Pose(coordPositionSubtract(lhs, rhs), coordOrientationSubtract(lhs.orientation, rhs.orientation));
-	}
+    //from http://osrf-distributions.s3.amazonaws.com/gazebo/api/dev/Pose_8hh_source.html
+    static Vector3T coordPositionSubtract(const Pose& lhs, const Pose& rhs)
+    {
+        QuaternionT tmp(0,
+            lhs.position.x() - rhs.position.x(),
+            lhs.position.y() - rhs.position.y(),
+            lhs.position.z() - rhs.position.z()
+        );
+
+        tmp = rhs.orientation.inverse() * (tmp * rhs.orientation);
+
+        return tmp.vec();
+    }
+    static QuaternionT coordOrientationSubtract(const QuaternionT& lhs, const QuaternionT& rhs)
+    {
+        QuaternionT result(rhs.inverse() * lhs);
+        result.normalize();
+        return result;
+    }
+    static Pose subtract(const Pose& lhs, const Pose& rhs)
+    {
+        return Pose(coordPositionSubtract(lhs, rhs), coordOrientationSubtract(lhs.orientation, rhs.orientation));
+    }
 
 
     static std::string toString(const Vector3T& vect, const char* prefix = nullptr)
@@ -370,19 +378,19 @@ class VectorMathT {
             return angle;
     }
 
-	/**
-	* \brief Extracts the yaw part from a quaternion, using RPY / euler (z-y'-z'') angles.
-	* RPY rotates about the fixed axes in the order x-y-z,
-	* which is the same as euler angles in the order z-y'-x''.
-	*/
-	static RealT yawFromQuaternion(const QuaternionT& q) {
-		return atan2(2.0 * (q.w() * q.z() + q.x() * q.y()),
-			1.0 - 2.0 * (q.y() * q.y() + q.z() * q.z()));
-	}
-
-	static QuaternionT quaternionFromYaw(RealT yaw) {
-		return QuaternionT(Eigen::AngleAxisd(yaw, Vector3T::UnitZ()));
-	}
+    /**
+    * \brief Extracts the yaw part from a quaternion, using RPY / euler (z-y'-z'') angles.
+    * RPY rotates about the fixed axes in the order x-y-z,
+    * which is the same as euler angles in the order z-y'-x''.
+    */
+    static RealT yawFromQuaternion(const QuaternionT& q) {
+        return atan2(2.0 * (q.w() * q.z() + q.x() * q.y()),
+            1.0 - 2.0 * (q.y() * q.y() + q.z() * q.z()));
+    }
+
+    static QuaternionT quaternionFromYaw(RealT yaw) {
+        return QuaternionT(Eigen::AngleAxisd(yaw, Vector3T::UnitZ()));
+    }
 };
 typedef VectorMathT<Eigen::Vector3d, Eigen::Quaternion<double,Eigen::DontAlign>, double> VectorMathd;
 typedef VectorMathT<Eigen::Vector3f, Eigen::Quaternion<float,Eigen::DontAlign>, float> VectorMathf;

Unreal/Environments/Blocks/Config/DefaultInput.ini

@@ -46,7 +46,7 @@ bEnableMouseSmoothing=True
 bEnableFOVScaling=True
 FOVScale=0.011110
 DoubleClickTime=0.200000
-bCaptureMouseOnLaunch=True
+bCaptureMouseOnLaunch=False
 DefaultViewportMouseCaptureMode=NoCapture
 bDefaultViewportMouseLock=True
 DefaultViewportMouseLockMode=DoNotLock

Unreal/Plugins/AirSim/Source/MultiRotorConnector.cpp

@@ -51,6 +51,11 @@ void MultiRotorConnector::initialize(AFlyingPawn* vehicle_pawn, msr::airlib::Mul
     memset(rotor_info_, 0, sizeof(RotorInfo) * rotor_count_);
 }
 
+msr::airlib::VehicleCameraBase* MultiRotorConnector::getCamera(unsigned int index)
+{
+    return camera_connectors_.at(index).get();
+}
+
 MultiRotorConnector::~MultiRotorConnector()
 {
     delete[] rotor_info_;

Unreal/Plugins/AirSim/Source/MultiRotorConnector.h

@@ -49,6 +49,8 @@ class MultiRotorConnector : public VehicleConnectorBase
     virtual void reportState(StateReporter& reporter) override;
     virtual UpdatableObject* getPhysicsBody() override;
 
+    virtual msr::airlib::VehicleCameraBase* getCamera(unsigned int index = 0) override;
+
 private:
     void detectUsbRc();
     static float joyStickToRC(int16_t val);

Unreal/Plugins/AirSim/Source/PIPCamera.cpp

@@ -14,6 +14,9 @@ void APIPCamera::PostInitializeComponents()
     screen_capture_ = UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("SceneCaptureComponent"));
     depth_capture_ = UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("DepthCaptureComponent"));
     seg_capture_ = UAirBlueprintLib::GetActorComponent<USceneCaptureComponent2D>(this, TEXT("SegmentationCaptureComponent"));
+
+    //set default for brigher images
+    scene_capture_settings_.target_gamma = CaptureSettings::kSceneTargetGamma;
 }
 
 void APIPCamera::BeginPlay()
@@ -26,19 +29,13 @@ void APIPCamera::BeginPlay()
 
     scene_render_target_ = NewObject<UTextureRenderTarget2D>();
     setCaptureSettings(ImageType_::Scene, scene_capture_settings_);
-    scene_render_target_->TargetGamma = 1.0f; // GEngine->GetDisplayGamma();
     //scene_render_target_->bHDR = false;
-    //scene_render_target_->InitAutoFormat(960, 540); //256 X 144, X 480
 
     depth_render_target_ = NewObject<UTextureRenderTarget2D>();
     setCaptureSettings(ImageType_::Depth, depth_capture_settings_);
-    depth_render_target_->TargetGamma = 1.0f;
-    //depth_render_target_->InitAutoFormat(960, 540);
 
     seg_render_target_ = NewObject<UTextureRenderTarget2D>();
     setCaptureSettings(ImageType_::Segmentation, seg_capture_settings_);
-    seg_render_target_->TargetGamma = 1.0f;
-    //seg_render_target_->InitAutoFormat(960, 540);
 }
 
 void APIPCamera::EndPlay(const EEndPlayReason::Type EndPlayReason)
@@ -118,13 +115,17 @@ void APIPCamera::setCaptureSettings(APIPCamera::ImageType_ type, const APIPCamer
 
 void APIPCamera::updateCaptureComponentSettings(USceneCaptureComponent2D* capture, UTextureRenderTarget2D* render_target, const CaptureSettings& settings)
 {
-    if (render_target)
+    if (render_target) {
         render_target->InitAutoFormat(settings.width, settings.height); //256 X 144, X 480
+        render_target->TargetGamma = settings.target_gamma;
+    }
     //else we will set this after this components get created
 
     if (capture) {
         capture->FOVAngle = settings.fov_degrees;
         capture->PostProcessSettings.AutoExposureSpeedDown = capture->PostProcessSettings.AutoExposureSpeedUp = settings.auto_exposure_speed;
+        capture->PostProcessSettings.AutoExposureMaxBrightness = capture->PostProcessSettings.AutoExposureMinBrightness = 1.0f;
+        capture->PostProcessSettings.AutoExposureBias = 1.0f;
         capture->PostProcessSettings.MotionBlurAmount = settings.motion_blur_amount;
     }
     //else we will set this after this components get created

Unreal/Plugins/AirSim/Source/PIPCamera.h

@@ -12,10 +12,16 @@ class AIRSIM_API APIPCamera : public ACameraActor
 
 public:
     struct CaptureSettings {
+        static constexpr float kSceneTargetGamma = 1.4f;
+
         unsigned int width = 256, height = 144; //960 X 540
         float fov_degrees = 90;
         float auto_exposure_speed = 100.0f;
+        float auto_exposure_bias = 1.0f;
+        float auto_exposure_max_brightness = 1.0f;
+        float auto_exposure_min_brightness = 1.0f;
         float motion_blur_amount = 0.0f;
+        float target_gamma = 1.0f; //should be defaulted to kSceneTargetGamma for scene
     };
 
 public:

Unreal/Plugins/AirSim/Source/Recording/RecordingFile.cpp

@@ -1,15 +1,22 @@
 #include <AirSim.h>
+#include <sstream>
 #include "ImageUtils.h"
 #include "common/ClockFactory.hpp"
 #include "common/common_utils/FileSystem.hpp"
 
-void RecordingFile::appendRecord(FString image_path, TArray<uint8>& compressedPng, const msr::airlib::PhysicsBody* physics_body)
+RecordingFile::RecordingFile()
+    : image_path_(FString(common_utils::FileSystem::getLogFileNamePath("img_", "", "", false).c_str())),
+    log_file_handle_(nullptr)
 {
-    if (compressedPng.Num() == 0)
+}
+
+void RecordingFile::appendRecord(TArray<uint8>& image_data, const msr::airlib::PhysicsBody* physics_body)
+{
+    if (image_data.Num() == 0)
         return;
 
-    FString filePath = image_path + FString::FromInt(images_saved_) + ".png";
-    bool imageSavedOk = FFileHelper::SaveArrayToFile(compressedPng, *filePath);
+    FString filePath = image_path_ + FString::FromInt(images_saved_) + ".png";
+    bool imageSavedOk = FFileHelper::SaveArrayToFile(image_data, *filePath);
 
     // If render command is complete, save image along with position and orientation
 
@@ -20,27 +27,56 @@ void RecordingFile::appendRecord(FString image_path, TArray<uint8>& compressedPn
 
         uint64_t timestamp_millis = static_cast<uint64_t>(msr::airlib::ClockFactory::get()->nowNanos() / 1.0E6);
 
-        record_file << timestamp_millis << "\t";
-        record_file << kinematics.pose.position.x() << "\t" << kinematics.pose.position.y() << "\t" << kinematics.pose.position.z() << "\t";
-        record_file << kinematics.pose.orientation.w() << "\t" << kinematics.pose.orientation.x() << "\t" << kinematics.pose.orientation.y() << "\t" << kinematics.pose.orientation.z() << "\t";
-        record_file << "\n";
+        std::stringstream ss;
+        ss << timestamp_millis << "\t";
+        ss << kinematics.pose.position.x() << "\t" << kinematics.pose.position.y() << "\t" << kinematics.pose.position.z() << "\t";
+        ss << kinematics.pose.orientation.w() << "\t" << kinematics.pose.orientation.x() << "\t" << kinematics.pose.orientation.y() << "\t" << kinematics.pose.orientation.z() << "\t";
+        ss << "\n";
+
+        writeLine(ss.str());
 
         UAirBlueprintLib::LogMessage(TEXT("Screenshot saved to:"), filePath, LogDebugLevel::Success);
         images_saved_++;
     }
 }
 
-void RecordingFile::startRecording()
+void RecordingFile::createFile(const std::string& file_path)
 {
-    if (record_file.is_open()) {
-        record_file.close();
-        UAirBlueprintLib::LogMessage(TEXT("Recording Error"), TEXT("File was already open"), LogDebugLevel::Failure);
-    }
+    closeFile();
+
+    IPlatformFile& platform_file = FPlatformFileManager::Get().GetPlatformFile();
+    log_file_handle_ = platform_file.OpenWrite(*FString(file_path.c_str()));
+}
+
+bool RecordingFile::isFileOpen()
+{
+    return log_file_handle_ != nullptr;
+}
+
+void RecordingFile::closeFile()
+{
+    if (isFileOpen())
+        delete log_file_handle_;
+
+    log_file_handle_ = nullptr;
+}
+
+void RecordingFile::writeLine(const std::string& line)
+{
+
+}
+
+RecordingFile::~RecordingFile()
+{
+    closeFile();
+}
 
+void RecordingFile::startRecording()
+{
     std::string fullPath = common_utils::FileSystem::getLogFileNamePath(record_filename, "", ".txt", true);
-    common_utils::FileSystem::createTextFile(fullPath, record_file);
+    createFile(fullPath);
 
-    if (record_file.is_open()) {
+    if (isFileOpen()) {
         is_recording_ = true;
 
         UAirBlueprintLib::LogMessage(TEXT("Recording"), TEXT("Started"), LogDebugLevel::Success);
@@ -52,11 +88,11 @@ void RecordingFile::startRecording()
 void RecordingFile::stopRecording()
 {
     is_recording_ = false;
-    if (!record_file.is_open()) {
+    if (isFileOpen()) {
         UAirBlueprintLib::LogMessage(TEXT("Recording Error"), TEXT("File was not open"), LogDebugLevel::Failure);
     }
     else
-        record_file.close();
+        closeFile();
 
     UAirBlueprintLib::LogMessage(TEXT("Recording"), TEXT("Stopped"), LogDebugLevel::Success);
 }