add

esim_py/CMakeLists.txt

@@ -0,0 +1,20 @@
+cmake_minimum_required(VERSION 3.3)
+project(esim_py)
+
+set(OpenCV_DIR "E:\\opencv\\build")
+
+find_package(pybind11 REQUIRED)
+find_package(OpenCV REQUIRED)
+find_package(Eigen3 REQUIRED NO_MODULE)
+find_package(Boost COMPONENTS system filesystem REQUIRED)
+
+set(CMAKE_POSITION_INDEPENDENT_CODE ON)
+set(CMAKE_CXX_STANDARD 11)
+
+include_directories(include ${EIGEN3_INCLUDE_DIR} ${OpenCV_INCLUDE_DIRS} ${Boost_INCLUDE_DIRS} ${PYTHON_INCLUDE_DIRS})
+
+add_library(libesim STATIC src/esim.cpp)
+
+pybind11_add_module(esim_py src/bindings.cpp)
+
+target_link_libraries(esim_py PRIVATE libesim ${OpenCV_LIBS} ${Boost_FILESYSTEM_LIBRARY} ${BOOST_SYSTEM_LIBRARY} Boost::filesystem Boost::system Eigen3::Eigen pybind11::module)

esim_py/README.md

@@ -0,0 +1,52 @@
+# esym\_py
+
+This package exposes python bindings for ESIM which can be used within a training loop. 
+To test out if the installation was successful you can run
+
+```bash
+python tests/test.py
+```
+
+which should print a message if completed sucessfully. 
+
+The currently supported functions are listed in the example below:
+```python
+import esim_py
+
+# constructor
+esim = esim_py.EventSimulator(
+    contrast_threshold_pos,  # contrast thesholds for positive 
+    contrast_threshold_neg,  # and negative events
+    refractory_period,  # minimum waiting period (in sec) before a pixel can trigger a new event
+    log_eps,  # epsilon that is used to numerical stability within the logarithm
+    use_log,  # wether or not to use log intensity
+    )
+
+# setter, useful within a training loop
+esim.setParameters(contrast_threshold_pos, contrast_threshold_neg, refractory_period, log_eps, use_log)
+
+# generate events from a sequence of images
+events_from_images = esim.generateFromFolder(
+    path_to_image_folder, # absolute path to folder that stores images in numbered order
+    path_to_timestamps    # absolute path to timestamps file containing one timestamp (in secs) for each 
+)
+
+# generate events from a video
+events_from_video = esim.generateFromVideo(
+    path_to_video_file,   # absolute path to video storing images
+    path_to_timestamps    # absolute path to timestamps file
+)
+
+# generate events from list of images and timestamps
+events_list_of_images = esim.generateFromStampedImageSequence(
+    list_of_image_files,   # list of absolute paths to images
+    list_of_timestamps     # list of timestamps in ascending order
+)
+
+```
+The example script `tests/plot_virtual_events.py` plots virtual events that are generated from images in `tests/data/images` with varying positive and negative contrast thresholds. To call it you need some additional pip packages:
+
+```bash
+pip install numpy matplotlib
+python tests/plot_virtual_events.py
+```

esim_py/include/.gitignore

@@ -0,0 +1 @@
+eigen3.3.7

esim_py/include/esim.h

@@ -0,0 +1,123 @@
+#pragma once
+
+#include <vector>
+
+#include <boost/filesystem.hpp>
+#include <Eigen/Core>
+#include <opencv2/core/core.hpp>
+
+#include <pybind11/numpy.h>
+#include <pybind11/pybind11.h>
+
+namespace py = pybind11;
+
+struct Event
+{
+  Event(int x, int y, double t, int polarity)
+  : x_(x), y_(y), t_(t), polarity_(polarity)
+  {}
+
+  bool operator<(Event& other)
+  {
+    return t_ < other.t_;
+  }
+
+  int x_, y_;
+  double t_;
+  int polarity_;
+};
+
+/*
+ * The EventSimulator takes as input a sequence of stamped images,
+ * assumed to be sampled at a "sufficiently high" framerate,
+ * and simulates the principle of operation of an idea event camera
+ * with a constant contrast threshold C.
+ * Pixel-wise intensity values are linearly interpolated in time.
+ *
+ * The pixel-wise voltages are reset with the values from the first image
+ * which is passed to the simulator.
+ */
+class EventSimulator
+{
+public:
+  EventSimulator(float contrast_threshold_pos, 
+                 float contrast_threshold_neg, 
+                 float refractory_period, 
+                 float log_eps,
+                 bool use_log_img);
+
+  Eigen::MatrixXd generateFromFolder(std::string image_folder, std::string timestamps_file_path);
+  Eigen::MatrixXd generateFromVideo(std::string video_path, std::string timestamps_file_path);
+  Eigen::MatrixXd generateFromStampedImageSequence(std::vector<std::string> image_paths, std::vector<double> timestamps);
+  
+  Eigen::MatrixXd generateEventFromCVImage(py::array_t<float> input_array, double time);
+  void initialise(py::array_t<float> input_array, double time);
+
+  void setParameters(float contrast_threshold_pos, 
+                     float contrast_threshold_neg,
+                     float refractory_period,
+                     float log_eps,
+                     bool use_log_img)
+  {
+    contrast_threshold_pos_ = contrast_threshold_pos;
+    contrast_threshold_neg_ = contrast_threshold_neg;
+    refractory_period_ = refractory_period;
+    log_eps_ = log_eps;
+    use_log_img_ = use_log_img;
+  }
+
+  cv::Mat pyArrayToCvMat(py::array_t<float> input_array) {
+    // Accessing the NumPy array data
+    py::buffer_info buf_info = input_array.request();
+    float* ptr = static_cast<float*>(buf_info.ptr);
+
+    // Assuming a 3-channel image, you may need to adjust channels and sizes accordingly
+    return cv::Mat(buf_info.shape[0], buf_info.shape[1], CV_32F, ptr).clone();
+}
+
+
+private:
+  void init(const cv::Mat &img, double time);  
+  Eigen::MatrixXd vec_to_eigen_matrix(std::vector<Event>& events_vec)
+  {
+    Eigen::MatrixXd events(events_vec.size(), 4);
+    for (int i=0; i<events_vec.size(); i++)
+    {
+        Event& event = events_vec[i]; 
+        events(i,0) = event.x_;
+        events(i,1) = event.y_;
+        events(i,2) = event.t_;
+        events(i,3) = event.polarity_;
+    }
+    return events;
+  }
+
+  void imageCallback(const cv::Mat& img, double time, std::vector<Event>& events);
+   
+  void read_directory_from_path(const std::string& name, std::vector<std::string>& v)
+  {
+      boost::filesystem::path p(name);
+      boost::filesystem::directory_iterator start(p);
+      boost::filesystem::directory_iterator end;
+
+      auto path_leaf_string = [](const boost::filesystem::directory_entry& entry) -> std::string {return entry.path().string();};
+
+      std::transform(start, end, std::back_inserter(v), path_leaf_string);
+
+      std::sort(v.begin(), v.end());
+  }  
+
+  float contrast_threshold_pos_; 
+  float contrast_threshold_neg_;
+  float refractory_period_;
+  float log_eps_;
+  bool use_log_img_;
+
+  bool is_initialized_;
+  double current_time_;
+  cv::Mat ref_values_;
+  cv::Mat last_img_;
+  cv::Mat last_event_timestamp_;
+  int image_height_;
+  int image_width_;
+};

esim_py/setup.py

@@ -0,0 +1,66 @@
+import os
+import platform
+import re
+import subprocess
+import sys
+
+from distutils.version import LooseVersion
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+
+class CMakeExtension(Extension):
+    def __init__(self, name, sourcedir=''):
+        super().__init__(name, sources=[])
+        self.sourcedir = os.path.abspath(sourcedir)
+
+
+class CMakeBuild(build_ext):
+    def run(self):
+        try:
+            out = subprocess.check_output(['cmake', '--version'])
+        except OSError:
+            raise RuntimeError("CMake must be installed to build the following extensions: " +
+                               ", ".join(e.name for e in self.extensions))
+
+        if platform.system() == "Windows":
+            cmake_version = LooseVersion(re.search(r'version\s*([\d.]+)', out.decode()).group(1))
+            if cmake_version < '3.1.0':
+                raise RuntimeError("CMake >= 3.1.0 is required on Windows")
+
+        for ext in self.extensions:
+            self.build_extension(ext)
+
+    def build_extension(self, ext):
+        extdir = os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name)))
+        cmake_args = ['-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
+                      '-DPYTHON_EXECUTABLE=' + sys.executable]
+
+        cfg = 'Debug' if self.debug else 'Release'
+        build_args = ['--config', cfg]
+
+        cmake_args += ['-DCMAKE_BUILD_TYPE=' + cfg]
+        build_args += ['--', '-j']
+
+        env = os.environ.copy()
+        env['CXXFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CXXFLAGS', ''),
+                                                              self.distribution.get_version())
+        env['CFLAGS'] = '{} -DVERSION_INFO=\\"{}\\"'.format(env.get('CFLAGS', ''),
+                                                            self.distribution.get_version())
+        if not os.path.exists(self.build_temp):
+            os.makedirs(self.build_temp)
+        subprocess.check_call(['cmake', ext.sourcedir] + cmake_args, cwd=self.build_temp, env=env)
+        subprocess.check_call(['cmake', '--build', '.'] + build_args, cwd=self.build_temp)
+
+
+
+setup(
+    name='esim_py',
+    version='0.0.1',
+    author='Daniel Gehrig',
+    author_email='[email protected]',
+    description='Python bindings for ESIM.',
+    long_description='',
+    ext_modules=[CMakeExtension('esim_py')],
+    cmdclass=dict(build_ext=CMakeBuild),
+    zip_safe=False,
+)

esim_py/src/bindings.cpp

@@ -0,0 +1,22 @@
+#include <esim.h>
+
+#include <pybind11/eigen.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+
+
+namespace py = pybind11;
+
+PYBIND11_MODULE(esim_py, m) {
+    m.doc() = "ESIM bindings";
+
+    py::class_<EventSimulator>(m, "EventSimulator")
+        .def(py::init<float,float,float,float,bool>())
+        .def("generateFromFolder", &EventSimulator::generateFromFolder, py::return_value_policy::reference_internal)
+        .def("generateFromVideo", &EventSimulator::generateFromVideo, py::return_value_policy::reference_internal)
+        .def("generateFromStampedImageSequence", &EventSimulator::generateFromStampedImageSequence, py::return_value_policy::reference_internal)
+        .def("setParameters", &EventSimulator::setParameters)
+        .def("generateEventFromCVImage", &EventSimulator::generateEventFromCVImage)
+        .def("init", &EventSimulator::initialise);
+
+}

esim_py/src/esim.cpp

@@ -0,0 +1,239 @@
+#include <esim.h>
+
+#include <fstream>
+#include <iostream>
+#include <algorithm>
+
+#include <opencv2/core/eigen.hpp>
+#include <opencv2/highgui/highgui.hpp>
+#include <opencv2/imgproc/imgproc.hpp>
+
+
+EventSimulator::EventSimulator(float contrast_threshold_pos, 
+                               float contrast_threshold_neg,
+                               float refractory_period,
+                               float log_eps,
+                               bool use_log_img)
+    : contrast_threshold_pos_(contrast_threshold_pos), contrast_threshold_neg_(contrast_threshold_neg),
+    refractory_period_(refractory_period), log_eps_(log_eps), use_log_img_(use_log_img), is_initialized_(false)
+{
+
+}
+
+Eigen::MatrixXd EventSimulator::generateFromVideo(std::string video_path, std::string timestamps_file_path)
+{
+    std::ifstream timestamps_file(timestamps_file_path);
+    
+    if(!timestamps_file.is_open()) 
+        throw std::runtime_error("unable to open the file " + timestamps_file_path);
+
+    cv::VideoCapture cap(video_path);
+
+    if ( !cap.isOpened() ) 
+        throw std::runtime_error("Cannot open the video file " + video_path);
+
+    std::string time_str;
+    double time;
+
+    std::vector<Event> events_vec;
+    
+    cv::Mat img, log_img;
+
+    while (cap.read(img))
+    {
+        img.convertTo(img, CV_32F, 1.0/255);
+        cv::Mat log_img = img;
+        if (use_log_img_)
+            cv::log(img+log_eps_, log_img);
+        
+        std::getline(timestamps_file, time_str);
+        time = std::stod(time_str);
+        
+        imageCallback(log_img, time, events_vec);
+    }
+
+    // reset state to generate new events
+    is_initialized_ = false;
+
+    return vec_to_eigen_matrix(events_vec);
+}
+
+Eigen::MatrixXd EventSimulator::generateFromStampedImageSequence(std::vector<std::string> image_paths, std::vector<double> timestamps)
+{
+    // check that timestamps are ascending
+    if (image_paths.size() != timestamps.size())
+        throw std::runtime_error("Number of image paths and number of timestamps should be equal. Got " + std::to_string(image_paths.size()) + " and " + std::to_string(timestamps.size()));
+
+    cv::Mat img, log_img;
+    double time;
+
+    std::vector<Event> events_vec;
+
+    for (int i=0; i<timestamps.size(); i++)
+    {
+        if ((i < timestamps.size()-1)  && timestamps[i+1]<timestamps[i]) 
+            throw std::runtime_error("Timestamps must be sorted in ascending order.");
+
+        img = cv::imread(image_paths[i], cv::IMREAD_GRAYSCALE);
+
+        if(img.empty()) 
+            throw std::runtime_error("unable to open the image " + image_paths[i]);
+       
+        img.convertTo(img, CV_32F, 1.0/255);
+        cv::Mat log_img = img;
+        if (use_log_img_)
+            cv::log(img+log_eps_, log_img);
+
+        time = timestamps[i];
+
+        imageCallback(log_img, time, events_vec);
+    }
+
+    // reset state to generate new events
+    is_initialized_ = false;
+
+    return vec_to_eigen_matrix(events_vec);
+}
+
+
+Eigen::MatrixXd EventSimulator::generateFromFolder(std::string image_folder, std::string timestamps_file_path)
+{
+    std::vector<std::string> image_files;
+    read_directory_from_path(image_folder, image_files);
+    std::ifstream timestamps_file(timestamps_file_path);
+    
+    if(!timestamps_file.is_open()) 
+        throw std::runtime_error("unable to open the file " + timestamps_file_path);
+
+    std::string time_str;
+    double time;
+
+    std::vector<Event> events_vec;
+    
+    cv::Mat img, log_img;
+
+    for (const std::string& file : image_files)
+    {
+        img = cv::imread(file, cv::IMREAD_GRAYSCALE);
+        if(img.empty()) 
+            throw std::runtime_error("unable to open the image " + file);
+
+        img.convertTo(img, CV_32F, 1.0/255);
+        cv::Mat log_img = img;
+        if (use_log_img_)
+            cv::log(img+log_eps_, log_img);
+
+        std::getline(timestamps_file, time_str);
+        time = std::stod(time_str);
+
+        imageCallback(log_img, time, events_vec);
+    }
+
+    // reset state to generate new events
+    is_initialized_ = false;
+
+    return vec_to_eigen_matrix(events_vec);
+}
+
+
+void EventSimulator::initialise(py::array_t<float> input_array, double time) {
+    cv::Mat img = pyArrayToCvMat(input_array);
+
+    init(img, time);
+}
+
+
+Eigen::MatrixXd EventSimulator::generateEventFromCVImage(py::array_t<float> input_array, double time) {
+    cv::Mat log_img = pyArrayToCvMat(input_array);
+
+    std::vector<Event> events_vec;
+
+    imageCallback(log_img, time, events_vec);
+
+    return vec_to_eigen_matrix(events_vec);
+}
+
+
+void EventSimulator::init(const cv::Mat &img, double time)
+{
+  is_initialized_ = true;
+  last_img_ = img;
+  ref_values_ = img;
+
+  last_event_timestamp_ = cv::Mat::zeros(img.size[0], img.size[1], CV_64F);
+
+  current_time_ = time;
+  image_width_ = img.size[1];
+  image_height_ = img.size[0];
+}
+
+
+void EventSimulator::imageCallback(const cv::Mat& img, double time, std::vector<Event>& events)
+{
+    cv::Mat preprocessed_img = img;
+  
+    if(!is_initialized_)
+    {
+        init(preprocessed_img, time);
+        return;
+    }
+
+    std::vector<Event> new_events;
+
+    static constexpr double kTolerance = 1e-6;
+    double delta_t = time - current_time_;
+
+    for (int y = 0; y < image_height_; ++y)
+    {
+        for (int x = 0; x < image_width_; ++x)
+        {
+            float& itdt = preprocessed_img.at<float>(y, x);
+            float& it = last_img_.at<float>(y, x);
+            float& prev_cross = ref_values_.at<float>(y, x);
+
+            if (std::fabs (it - itdt) > kTolerance)
+            {
+                float pol = (itdt >= it) ? +1.0 : -1.0;
+                float C = (pol > 0) ? contrast_threshold_pos_ : contrast_threshold_neg_;
+                
+                float curr_cross = prev_cross;
+                bool all_crossings = false;
+      
+                do
+                {
+                    curr_cross += pol * C;
+      
+                    if ((pol > 0 && curr_cross > it && curr_cross <= itdt)
+                        || (pol < 0 && curr_cross < it && curr_cross >= itdt))
+                    {
+                        const double edt = (curr_cross - it) * delta_t / (itdt - it);
+                        const double t = current_time_ + edt;
+      
+                        const double last_stamp_at_xy = last_event_timestamp_.at<double>(y,x);
+                        
+                        const double dt = t - last_stamp_at_xy;
+
+                        if(last_stamp_at_xy == 0 || dt >= refractory_period_)
+                        {
+                            new_events.emplace_back(x, y,t,pol);
+                            last_event_timestamp_.at<double>(y,x) = t;
+                        }
+                     
+                        ref_values_.at<float>(y,x) = curr_cross;
+                    }
+                    else
+                    {
+                        all_crossings = true;
+                    }
+                } while (!all_crossings);
+            } // end tolerance
+        } // end for each pixel
+    }
+
+    current_time_ = time;
+    last_img_ = preprocessed_img; // it is now the latest image
+
+    // need to sort the new events before inserting
+    std::sort(new_events.begin(), new_events.end());
+    events.insert(events.end(), new_events.begin(), new_events.end());
+}