initial commit

This commit is contained in:
Henri Rebecq
2018-10-29 17:53:15 +01:00
commit a8c2f0ca43
208 changed files with 554184 additions and 0 deletions

View File

@ -0,0 +1,30 @@
cmake_minimum_required(VERSION 2.8.3)
project(esim)
find_package(catkin_simple REQUIRED)
find_package(OpenCV REQUIRED)
catkin_simple()
set(HEADERS
include/esim/esim/simulator.hpp
include/esim/esim/event_simulator.hpp
include/esim/esim/camera_simulator.hpp
)
set(SOURCES
src/simulator.cpp
src/event_simulator.cpp
src/camera_simulator.cpp
)
cs_add_library(${PROJECT_NAME} ${SOURCES} ${HEADERS})
##########
# GTESTS #
##########
catkin_add_gtest(test_event_simulator test/test_event_simulator.cpp)
target_link_libraries(test_event_simulator ${PROJECT_NAME} ${OpenCV_LIBRARIES})
cs_install()
cs_export()

View File

@ -0,0 +1,72 @@
#pragma once
#include <esim/common/types.hpp>
#include <deque>
#include <ze/common/time_conversions.hpp>
namespace event_camera_simulator {
class ImageBuffer
{
public:
ZE_POINTER_TYPEDEFS(ImageBuffer);
struct ImageData
{
ImageData(Image img, Time stamp, Duration exposure_time)
: image(img),
stamp(stamp),
exposure_time(exposure_time) {}
Image image;
Time stamp;
Duration exposure_time; // timestamp since last image (0 if this is the first image)
};
using ExposureImage = std::pair<Duration, Image>;
// Rolling image buffer of mazimum size 'buffer_size_ns'.
ImageBuffer(Duration buffer_size_ns)
: buffer_size_ns_(buffer_size_ns) {}
void addImage(Time t, const Image& img);
std::deque<ImageData> getRawBuffer() const { return data_; }
size_t size() const { return data_.size(); }
Duration getExposureTime() const { return buffer_size_ns_; }
private:
Duration buffer_size_ns_;
std::deque<ImageData> data_;
};
/*
* The CameraSimulator takes as input a sequence of stamped images,
* assumed to be sampled at a "sufficiently high" framerate and with
* floating-point precision, and treats each image as a measure of
* irradiance.
* From this, it simulates a real camera, including motion blur.
*
* @TODO: simulate a non-linear camera response curve, shot noise, etc.
*/
class CameraSimulator
{
public:
CameraSimulator(double exposure_time_ms)
: exposure_time_(ze::secToNanosec(exposure_time_ms / 1000.0))
{
buffer_.reset(new ImageBuffer(exposure_time_));
}
bool imageCallback(const Image& img, Time time,
const ImagePtr &camera_image);
private:
ImageBuffer::Ptr buffer_;
const Duration exposure_time_;
};
} // namespace event_camera_simulator

View File

@ -0,0 +1,54 @@
#pragma once
#include <esim/common/types.hpp>
namespace event_camera_simulator {
/*
* The EventSimulator takes as input a sequence of stamped images,
* assumed to be sampled at a "sufficiently high" framerate,
* and simulates the principle of operation of an idea event camera
* with a constant contrast threshold C.
* Pixel-wise intensity values are linearly interpolated in time.
*
* The pixel-wise voltages are reset with the values from the first image
* which is passed to the simulator.
*/
class EventSimulator
{
public:
struct Config
{
double Cp;
double Cm;
double sigma_Cp;
double sigma_Cm;
Duration refractory_period_ns;
bool use_log_image;
double log_eps;
};
using TimestampImage = cv::Mat_<ze::real_t>;
EventSimulator(const Config& config)
: config_(config),
is_initialized_(false),
current_time_(0)
{}
void init(const Image& img, Time time);
Events imageCallback(const Image& img, Time time);
private:
bool is_initialized_;
Time current_time_;
Image ref_values_;
Image last_img_;
TimestampImage last_event_timestamp_;
cv::Size size_;
Config config_;
};
} // namespace event_camera_simulator

View File

@ -0,0 +1,59 @@
#pragma once
#include <esim/esim/event_simulator.hpp>
#include <esim/esim/camera_simulator.hpp>
#include <esim/visualization/publisher_interface.hpp>
namespace event_camera_simulator {
/* The Simulator forwards the simulated images / depth maps
* from the data provider to multiple, specialized camera simulators, such as:
* (i) event camera simulators that simulate events based on sequences of images
* (ii) camera simulators that simulate real cameras
* (including motion blur, camera response function, noise, etc.)
*
* The Simulator then forwards the simulated data to one or more publishers.
*/
class Simulator
{
public:
Simulator(size_t num_cameras,
const EventSimulator::Config& event_sim_config,
double exposure_time_ms)
: num_cameras_(num_cameras),
exposure_time_(ze::millisecToNanosec(exposure_time_ms))
{
for(size_t i=0; i<num_cameras_; ++i)
{
event_simulators_.push_back(EventSimulator(event_sim_config));
camera_simulators_.push_back(CameraSimulator(exposure_time_ms));
}
}
~Simulator();
void addPublisher(const Publisher::Ptr& publisher)
{
CHECK(publisher);
publishers_.push_back(std::move(publisher));
}
void dataProviderCallback(const SimulatorData& sim_data);
void publishData(const SimulatorData &sim_data,
const EventsVector &events,
const ImagePtrVector &camera_images);
private:
size_t num_cameras_;
std::vector<EventSimulator> event_simulators_;
std::vector<CameraSimulator> camera_simulators_;
Duration exposure_time_;
std::vector<Publisher::Ptr> publishers_;
ImagePtrVector corrupted_camera_images_;
};
} // namespace event_camera_simulator

View File

@ -0,0 +1,20 @@
<?xml version="1.0"?>
<package format="2">
<name>esim</name>
<version>0.0.0</version>
<description>Event camera simulator, which can simulate events from a stream of images samplet at high frequency.</description>
<maintainer email="rebecq@ifi.uzh.ch">Henri Rebecq</maintainer>
<license>GPLv3</license>
<buildtool_depend>catkin</buildtool_depend>
<buildtool_depend>catkin_simple</buildtool_depend>
<depend>esim_common</depend>
<depend>esim_data_provider</depend>
<depend>esim_visualization</depend>
<depend>gflags_catkin</depend>
<depend>glog_catkin</depend>
</package>

View File

@ -0,0 +1,62 @@
#include <esim/esim/camera_simulator.hpp>
namespace event_camera_simulator {
void ImageBuffer::addImage(Time t, const Image& img)
{
if(!data_.empty())
{
// Check that the image timestamps are monotonically increasing
CHECK_GT(t, data_.back().stamp);
}
Duration exposure_time = data_.empty() ? 0 : t - data_.back().stamp;
VLOG(2) << "Adding image to buffer with stamp: " << t
<< " and exposure time " << exposure_time;
data_.push_back(ImageData(img.clone(), t, exposure_time));
// Remove all the images with timestamp older than t - buffer_size_ns_
auto first_valid_element = std::lower_bound(data_.begin(), data_.end(), t - buffer_size_ns_,
[](ImageData lhs, Time rhs) -> bool { return lhs.stamp < rhs; });
data_.erase(data_.begin(), first_valid_element);
VLOG(3) << "first/last element in buffer: "
<< data_.front().stamp
<< " " << data_.back().stamp;
VLOG(3) << "number of images in the buffer: " << data_.size();
CHECK_LE(data_.back().stamp - data_.front().stamp, buffer_size_ns_);
}
bool CameraSimulator::imageCallback(const Image &img, Time time,
const ImagePtr& camera_image)
{
CHECK(camera_image);
CHECK_EQ(camera_image->size(), img.size());
buffer_->addImage(time, img);
static const Time initial_time = time;
if(time - initial_time < exposure_time_)
{
LOG_FIRST_N(WARNING, 1) << "The images do not cover a time span long enough to simulate the exposure time accurately.";
return false;
}
// average all the images in the buffer to simulate motion blur
camera_image->setTo(0);
ze::real_t denom = 0.;
for(const ImageBuffer::ImageData& img : buffer_->getRawBuffer())
{
*camera_image += ze::nanosecToMillisecTrunc(img.exposure_time) * img.image;
denom += ze::nanosecToMillisecTrunc(img.exposure_time);
}
*camera_image /= denom;
cv::Mat disp;
camera_image->convertTo(disp, CV_8U, 255);
return true;
}
} // namespace event_camera_simulator

View File

@ -0,0 +1,117 @@
#include <esim/esim/event_simulator.hpp>
#include <ze/common/random.hpp>
#include <glog/logging.h>
#include <opencv2/imgproc/imgproc.hpp>
#include <ze/common/time_conversions.hpp>
namespace event_camera_simulator {
void EventSimulator::init(const Image &img, Time time)
{
VLOG(1) << "Initialized event camera simulator with sensor size: " << img.size();
VLOG(1) << "and contrast thresholds: C+ = " << config_.Cp << " , C- = " << config_.Cm;
is_initialized_ = true;
last_img_ = img.clone();
ref_values_ = img.clone();
last_event_timestamp_ = TimestampImage::zeros(img.size());
current_time_ = time;
size_ = img.size();
}
Events EventSimulator::imageCallback(const Image& img, Time time)
{
CHECK_GE(time, 0);
Image preprocessed_img = img.clone();
if(config_.use_log_image)
{
LOG_FIRST_N(INFO, 1) << "Converting the image to log image with eps = " << config_.log_eps << ".";
cv::log(config_.log_eps + img, preprocessed_img);
}
if(!is_initialized_)
{
init(preprocessed_img, time);
return {};
}
// For each pixel, check if new events need to be generated since the last image sample
static constexpr ImageFloatType tolerance = 1e-6;
Events events;
Duration delta_t_ns = time - current_time_;
CHECK_GT(delta_t_ns, 0u);
CHECK_EQ(img.size(), size_);
for (int y = 0; y < size_.height; ++y)
{
for (int x = 0; x < size_.width; ++x)
{
ImageFloatType itdt = preprocessed_img(y, x);
ImageFloatType it = last_img_(y, x);
ImageFloatType prev_cross = ref_values_(y, x);
if (std::fabs (it - itdt) > tolerance)
{
ImageFloatType pol = (itdt >= it) ? +1.0 : -1.0;
ImageFloatType C = (pol > 0) ? config_.Cp : config_.Cm;
ImageFloatType sigma_C = (pol > 0) ? config_.sigma_Cp : config_.sigma_Cm;
if(sigma_C > 0)
{
C += ze::sampleNormalDistribution<ImageFloatType>(false, 0, sigma_C);
}
ImageFloatType curr_cross = prev_cross;
bool all_crossings = false;
do
{
curr_cross += pol * C;
if ((pol > 0 && curr_cross > it && curr_cross <= itdt)
|| (pol < 0 && curr_cross < it && curr_cross >= itdt))
{
Duration edt = (curr_cross - it) * delta_t_ns / (itdt - it);
Time t = current_time_ + edt;
// check that pixel (x,y) is not currently in a "refractory" state
// i.e. |t-that last_timestamp(x,y)| >= refractory_period
const Time last_stamp_at_xy = ze::secToNanosec(last_event_timestamp_(y,x));
CHECK_GE(t, last_stamp_at_xy);
const Duration dt = t - last_stamp_at_xy;
if(last_event_timestamp_(y,x) == 0 || dt >= config_.refractory_period_ns)
{
events.push_back(Event(x, y, t, pol > 0));
last_event_timestamp_(y,x) = ze::nanosecToSecTrunc(t);
}
else
{
VLOG(3) << "Dropping event because time since last event ("
<< dt << " ns) < refractory period ("
<< config_.refractory_period_ns << " ns).";
}
ref_values_(y,x) = curr_cross;
}
else
{
all_crossings = true;
}
} while (!all_crossings);
} // end tolerance
} // end for each pixel
}
// update simvars for next loop
current_time_ = time;
last_img_ = preprocessed_img.clone(); // it is now the latest image
// Sort the events by increasing timestamps, since this is what
// most event processing algorithms expect
sort(events.begin(), events.end(),
[](const Event& a, const Event& b) -> bool
{
return a.t < b.t;
});
return events;
}
} // namespace event_camera_simulator

View File

@ -0,0 +1,137 @@
#include <esim/esim/simulator.hpp>
#include <ze/common/timer_collection.hpp>
#include <esim/common/utils.hpp>
namespace event_camera_simulator {
DECLARE_TIMER(TimerEventSimulator, timers_event_simulator_,
simulate_events,
visualization
);
Simulator::~Simulator()
{
timers_event_simulator_.saveToFile("/tmp", "event_simulator.csv");
}
void Simulator::dataProviderCallback(const SimulatorData &sim_data)
{
CHECK_EQ(event_simulators_.size(), num_cameras_);
if(sim_data.images_updated)
{
EventsVector events(num_cameras_);
Time time = sim_data.timestamp;
// simulate the events and camera images for every sensor in the rig
{
auto t = timers_event_simulator_[TimerEventSimulator::simulate_events].timeScope();
for(size_t i=0; i<num_cameras_; ++i)
{
events[i] = event_simulators_[i].imageCallback(*sim_data.images[i], time);
if(corrupted_camera_images_.size() < num_cameras_)
{
// allocate memory for the corrupted camera images and set them to 0
corrupted_camera_images_.emplace_back(std::make_shared<Image>(sim_data.images[i]->size()));
corrupted_camera_images_[i]->setTo(0.);
}
camera_simulators_[i].imageCallback(*sim_data.images[i], time, corrupted_camera_images_[i]);
}
}
// publish the simulation data + events
{
auto t = timers_event_simulator_[TimerEventSimulator::visualization].timeScope();
publishData(sim_data, events, corrupted_camera_images_);
}
}
else
{
{
// just forward the simulation data to the publisher
auto t = timers_event_simulator_[TimerEventSimulator::visualization].timeScope();
publishData(sim_data, {}, corrupted_camera_images_);
}
}
}
void Simulator::publishData(const SimulatorData& sim_data,
const EventsVector& events,
const ImagePtrVector& camera_images)
{
if(publishers_.empty())
{
LOG_FIRST_N(WARNING, 1) << "No publisher available";
return;
}
Time time = sim_data.timestamp;
const Transformation& T_W_B = sim_data.groundtruth.T_W_B;
const TransformationVector& T_W_Cs = sim_data.groundtruth.T_W_Cs;
const ze::CameraRig::Ptr& camera_rig = sim_data.camera_rig;
// Publish the new data (events, images, depth maps, poses, point clouds, etc.)
if(!events.empty())
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->eventsCallback(events);
}
if(sim_data.poses_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->poseCallback(T_W_B, T_W_Cs, time);
}
if(sim_data.twists_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->twistCallback(sim_data.groundtruth.angular_velocities_,
sim_data.groundtruth.linear_velocities_,
time);
}
if(sim_data.imu_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->imuCallback(sim_data.specific_force_corrupted, sim_data.angular_velocity_corrupted, time);
}
if(camera_rig)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->cameraInfoCallback(camera_rig, time);
}
if(sim_data.images_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
{
publisher->imageCallback(sim_data.images, time);
// the images should be timestamped at mid-exposure (unless it is not possible)
const Time mid_exposure_time = (time >= 0.5 * exposure_time_) ? time - 0.5 * exposure_time_ : time;
publisher->imageCorruptedCallback(camera_images, mid_exposure_time);
}
}
if(sim_data.depthmaps_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->depthmapCallback(sim_data.depthmaps, time);
}
if(sim_data.optic_flows_updated)
{
for(const Publisher::Ptr& publisher : publishers_)
publisher->opticFlowCallback(sim_data.optic_flows, time);
}
if(sim_data.depthmaps_updated && !events.empty())
{
PointCloudVector pointclouds(num_cameras_);
for(size_t i=0; i<num_cameras_; ++i)
{
CHECK(sim_data.depthmaps[i]);
pointclouds[i] = eventsToPointCloud(events[i], *sim_data.depthmaps[i], camera_rig->atShared(i));
}
for(const Publisher::Ptr& publisher : publishers_)
publisher->pointcloudCallback(pointclouds, time);
}
}
} // namespace event_camera_simulator

View File

@ -0,0 +1,237 @@
#include <esim/esim/event_simulator.hpp>
#include <ze/common/test_entrypoint.hpp>
#include <fstream>
#include <ze/common/file_utils.hpp>
#include <ze/common/path_utils.hpp>
#include <ze/common/string_utils.hpp>
#include <ze/common/file_utils.hpp>
#include <ze/common/time_conversions.hpp>
#include <ze/matplotlib/matplotlibcpp.hpp>
#include <opencv2/highgui/highgui.hpp>
#define USE_OPENCV
namespace event_camera_simulator
{
class CSVImageLoader
{
public:
CSVImageLoader(const std::string& path_to_data_folder)
: path_to_data_folder_(path_to_data_folder)
{
images_in_str_.open(ze::joinPath(path_to_data_folder, "images.csv"));
CHECK(images_in_str_.is_open());
}
bool next(int64_t& stamp, Image& img)
{
std::string line;
if(!getline(images_in_str_, line))
{
LOG(INFO) << "Finished reading all the images in the folder";
return false;
}
if('%' != line.at(0) && '#' != line.at(0))
{
std::vector<std::string> items = ze::splitString(line, delimiter_);
stamp = std::stoll(items[0]);
const std::string& path_to_image
= ze::joinPath(path_to_data_folder_, "frame", "cam_0", items[1]);
img = cv::imread(path_to_image, 0);
CHECK(img.data) << "Error loading image: " << path_to_image;
return true;
}
else
{
return next(stamp, img);
}
}
private:
std::ifstream images_in_str_;
const char delimiter_{','};
std::string path_to_data_folder_;
};
} // namespace event_camera_simulator
std::string getTestDataDir(const std::string& dataset_name)
{
using namespace ze;
const char* datapath_dir = std::getenv("ESIM_TEST_DATA_PATH");
CHECK(datapath_dir != nullptr)
<< "Did you download the esim_test_data repository and set "
<< "the ESIM_TEST_DATA_PATH environment variable?";
std::string path(datapath_dir);
CHECK(isDir(path)) << "Folder does not exist: " << path;
path = path + "/data/" + dataset_name;
CHECK(isDir(path)) << "Dataset does not exist: " << path;
return path;
}
TEST(EventSimulator, testImageReconstruction)
{
using namespace event_camera_simulator;
// Load image sequence from folder
const std::string path_to_data_folder = getTestDataDir("planar_carpet");
CSVImageLoader reader(path_to_data_folder);
EventSimulator::Config event_sim_config;
event_sim_config.Cp = 0.05;
event_sim_config.Cm = 0.03;
event_sim_config.sigma_Cp = 0;
event_sim_config.sigma_Cm = 0;
event_sim_config.use_log_image = true;
event_sim_config.log_eps = 0.001;
EventSimulator simulator(event_sim_config);
LOG(INFO) << "Testing event camera simulator with C+ = " << event_sim_config.Cp
<< ", C- = " << event_sim_config.Cm;
const ImageFloatType max_reconstruction_error
= std::max(event_sim_config.Cp, event_sim_config.Cm);
bool is_first_image = true;
Image I, L, L_reconstructed;
int64_t stamp;
while(reader.next(stamp, I))
{
I.convertTo(I, cv::DataType<ImageFloatType>::type, 1./255.);
cv::log(event_sim_config.log_eps + I, L);
if(is_first_image)
{
// Initialize reconstructed image from the ground truth image
L_reconstructed = L.clone();
is_first_image = false;
}
Events events = simulator.imageCallback(I, stamp);
// Reconstruct next image from previous one using the events in between
for(const Event& e : events)
{
ImageFloatType pol = e.pol ? 1. : -1.;
const ImageFloatType C = e.pol ? event_sim_config.Cp : event_sim_config.Cm;
L_reconstructed(e.y,e.x) += pol * C;
}
// Check that the reconstruction error is bounded by the contrast thresholds
for(int y=0; y<I.rows; ++y)
{
for(int x=0; x<I.cols; ++x)
{
const ImageFloatType reconstruction_error = std::fabs(L_reconstructed(y,x) - L(y,x));
VLOG_EVERY_N(2, I.rows * I.cols) << reconstruction_error;
EXPECT_LE(reconstruction_error, max_reconstruction_error);
}
}
#ifdef USE_OPENCV
const ImageFloatType vmin = std::log(event_sim_config.log_eps);
const ImageFloatType vmax = std::log(1.0 + event_sim_config.log_eps);
cv::Mat disp = 255.0 * (L_reconstructed - vmin) / (vmax - vmin);
disp.convertTo(disp, CV_8U);
cv::imshow("Reconstructed", disp);
cv::waitKey(1);
#endif
}
}
TEST(EventSimulator, testEvolutionReconstructionError)
{
using namespace event_camera_simulator;
// Load image sequence from folder
const std::string path_to_data_folder = getTestDataDir("planar_carpet");
CSVImageLoader reader(path_to_data_folder);
EventSimulator::Config event_sim_config;
event_sim_config.Cp = 0.5;
event_sim_config.Cm = event_sim_config.Cp;
event_sim_config.sigma_Cp = 0;
event_sim_config.sigma_Cm = event_sim_config.sigma_Cp;
event_sim_config.use_log_image = true;
event_sim_config.log_eps = 0.001;
EventSimulator simulator(event_sim_config);
const double contrast_bias = 0.0;
LOG(INFO) << "Testing event camera simulator with C+ = " << event_sim_config.Cp
<< ", C- = " << event_sim_config.Cm;
std::vector<ze::real_t> times, mean_errors, stddev_errors;
bool is_first_image = true;
Image I, L, L_reconstructed;
int64_t stamp;
while(reader.next(stamp, I))
{
LOG_EVERY_N(INFO, 50) << "t = " << ze::nanosecToSecTrunc(stamp) << " s";
I.convertTo(I, cv::DataType<ImageFloatType>::type, 1./255.);
cv::log(event_sim_config.log_eps + I, L);
if(is_first_image)
{
// Initialize reconstructed image from the ground truth image
L_reconstructed = L.clone();
is_first_image = false;
}
Events events = simulator.imageCallback(I, stamp);
// Reconstruct next image from previous one using the events in between
for(const Event& e : events)
{
ImageFloatType pol = e.pol ? 1. : -1.;
ImageFloatType C = e.pol ? event_sim_config.Cp : event_sim_config.Cm;
C += contrast_bias;
L_reconstructed(e.y,e.x) += pol * C;
}
// Compute the mean and average reconstruction error over the whole image
Image error;
cv::absdiff(L, L_reconstructed, error);
cv::Scalar mean_error, stddev_error;
cv::meanStdDev(error, mean_error, stddev_error);
VLOG(1) << "Mean error: " << mean_error
<< ", Stddev: " << stddev_error;
times.push_back(ze::nanosecToSecTrunc(stamp));
mean_errors.push_back(mean_error[0]);
stddev_errors.push_back(stddev_error[0]);
#ifdef USE_OPENCV
const ImageFloatType vmin = std::log(event_sim_config.log_eps);
const ImageFloatType vmax = std::log(1.0 + event_sim_config.log_eps);
cv::Mat disp = 255.0 * (L_reconstructed - vmin) / (vmax - vmin);
disp.convertTo(disp, CV_8U);
cv::imshow("Reconstructed", disp);
cv::waitKey(1);
#endif
}
// Plot the mean and stddev reconstruction error over time
ze::plt::plot(times, mean_errors, "r");
ze::plt::plot(times, stddev_errors, "b--");
std::stringstream title;
title << "C = " << event_sim_config.Cp
<< ", sigma = " << event_sim_config.sigma_Cp
<< ", bias = " << contrast_bias;
ze::plt::title(title.str());
ze::plt::xlabel("Time (s)");
ze::plt::ylabel("Mean / Stddev reconstruction error");
ze::plt::grid(true);
ze::plt::save("/tmp/evolution_reconstruction_error.pdf");
ze::plt::show();
}
ZE_UNITTEST_ENTRYPOINT