From 19a738f07b99ef255e475d40a2785a6b82cba8f7 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sat, 1 Apr 2017 20:55:23 -0400 Subject: [PATCH 01/19] Allow setting flow for initializing NovelViewGenerator w/o prepare --- surround360_render/source/optical_flow/NovelView.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/surround360_render/source/optical_flow/NovelView.h b/surround360_render/source/optical_flow/NovelView.h index eb6f97e5..af5cc632 100644 --- a/surround360_render/source/optical_flow/NovelView.h +++ b/surround360_render/source/optical_flow/NovelView.h @@ -135,6 +135,9 @@ class NovelViewGenerator { // for debugging virtual Mat getFlowLtoR() { return Mat(); } virtual Mat getFlowRtoL() { return Mat(); } + + virtual void setFlowLtoR(const Mat& flow) = 0; + virtual void setFlowRtoL(const Mat& flow) = 0; }; // this is a base class for novel view generators that work by reduction to optical flow. @@ -164,6 +167,9 @@ class NovelViewGeneratorLazyFlow : public NovelViewGenerator { Mat getFlowLtoR() { return flowLtoR; } Mat getFlowRtoL() { return flowRtoL; } + + void setFlowLtoR(const Mat& flow) { flowLtoR = flow; } + void setFlowRtoL(const Mat& flow) { flowRtoL = flow; } }; // the name "asymmetric" here refers to the idea that we compute an optical flow from From 380be244a910fbca6fd6344c5a01532fd86f3d6f Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Thu, 6 Apr 2017 19:10:26 -0400 Subject: [PATCH 02/19] Fixup build issues --- surround360_render/CMakeLists.txt | 16 ++++++++++++---- .../scripts/geometric_calibration.py | 1 + 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index 1ee7c391..95cfd13e 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -15,6 +15,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/render) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}) INCLUDE_DIRECTORIES(/usr/local/include/eigen3) +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # required for clang, including AppleClang LINK_DIRECTORIES(/usr/local/lib) @@ -90,8 +92,7 @@ FILE(GLOB optical_flow_SRC "source/optical_flow/*.cpp") FILE(GLOB render_SRC "source/render/*.cpp") FILE(GLOB util_SRC "source/util/*.cpp") -ADD_LIBRARY( - LibVrCamera +ADD_LIBRARY(LibVrCamera source/util/SystemUtil.cpp ${calibration_SRC} ${optical_flow_SRC} @@ -198,6 +199,12 @@ TARGET_LINK_LIBRARIES( LibJSON glog gflags + +# folly +# ceres +# LibJSON + ${OpenCV_LIBS} + ${PLATFORM_SPECIFIC_LIBS} ) ### TestOpticalFlow ### @@ -498,11 +505,12 @@ TARGET_LINK_LIBRARIES( LibJSON gflags glog - folly boost_filesystem boost_system double-conversion + folly + ${CERES_LIBRARIES} ${OpenCV_LIBS} ${PLATFORM_SPECIFIC_LIBS} - ${CERES_LIBRARIES} ) + diff --git a/surround360_render/scripts/geometric_calibration.py b/surround360_render/scripts/geometric_calibration.py index ead2034d..cb907b50 100644 --- a/surround360_render/scripts/geometric_calibration.py +++ b/surround360_render/scripts/geometric_calibration.py @@ -36,6 +36,7 @@ {COLMAP_DIR}/feature_extractor --General.image_path {IMAGE_PATH} --General.database_path {COLMAP_DB_PATH} +--ExtractionOptions.gpu_index 0 """ COLMAP_MATCH_TEMPLATE = """ From d27a7d8f0902acb7bea9aa3a9e4c593fc0d6eb12 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Thu, 6 Apr 2017 19:10:42 -0400 Subject: [PATCH 03/19] Enable rig being constructed without a filename --- surround360_render/source/render/RigDescription.cpp | 7 +++++-- surround360_render/source/render/RigDescription.h | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/surround360_render/source/render/RigDescription.cpp b/surround360_render/source/render/RigDescription.cpp index 8864906e..f68162c3 100644 --- a/surround360_render/source/render/RigDescription.cpp +++ b/surround360_render/source/render/RigDescription.cpp @@ -15,8 +15,11 @@ using namespace cv; using namespace std; using namespace surround360::util; -RigDescription::RigDescription(const string& filename) { - rig = Camera::loadRig(filename); +RigDescription::RigDescription(const string& filename) + : RigDescription(Camera::loadRig(filename)) { +} + +RigDescription::RigDescription(const Camera::Rig& rig) { for (const Camera& camera : rig) { if (camera.group.find("side") != string::npos) { rigSideOnly.emplace_back(camera); diff --git a/surround360_render/source/render/RigDescription.h b/surround360_render/source/render/RigDescription.h index a452e188..0d5ab739 100644 --- a/surround360_render/source/render/RigDescription.h +++ b/surround360_render/source/render/RigDescription.h @@ -29,6 +29,8 @@ struct RigDescription { Camera::Rig rigSideOnly; RigDescription(const string& filename); + RigDescription(const Camera::Rig& rig); + // find the camera that is closest to pointing in the provided direction // ignore those with excessive distance from the camera axis to the rig center const Camera& findCameraByDirection( From 1a2083a626bcc30234966ddc675edbddd9362661 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Thu, 6 Apr 2017 23:40:05 -0400 Subject: [PATCH 04/19] Working spherical projection kernel in scanner --- surround360_render/CMakeLists.txt | 21 ++ .../scripts/scanner_process_video.py | 264 ++++++++++++++++++ .../project_spherical_kernel_cpu.cpp | 146 ++++++++++ .../source/scanner_kernels/surround360.proto | 9 + 4 files changed, 440 insertions(+) create mode 100644 surround360_render/scripts/scanner_process_video.py create mode 100644 surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp create mode 100644 surround360_render/source/scanner_kernels/surround360.proto diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index 95cfd13e..a09618d2 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -2,6 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.2) PROJECT(Surround360Render CXX) +OPTION(USE_SCANNER "Enable using Scanner for distributed rendering" ON) + FIND_PACKAGE(OpenCV) FIND_PACKAGE(Ceres REQUIRED) @@ -99,6 +101,12 @@ ADD_LIBRARY(LibVrCamera ${render_SRC} ${util_SRC} ) +# TARGET_LINK_LIBRARIES(LibVrCamera +# folly +# ceres +# LibJSON +# ${OpenCV_LIBS} +# ${PLATFORM_SPECIFIC_LIBS}) TARGET_COMPILE_FEATURES(LibVrCamera PRIVATE cxx_range_for) ### Raw2Rgb ### @@ -514,3 +522,16 @@ TARGET_LINK_LIBRARIES( ${PLATFORM_SPECIFIC_LIBS} ) + +IF (USE_SCANNER) + SET(SCANNER_PATH "~/repos/scanner/") + include(${SCANNER_PATH}/cmake/Util/Op.cmake) + build_op( + LIB_NAME surround360kernels + CPP_SRCS source/scanner_kernels/project_spherical_kernel_cpu.cpp + PROTO_SRC source/scanner_kernels/surround360.proto) + target_link_libraries(surround360kernels PUBLIC + LibVrCamera + folly + ${OpenCV_LIBS}) +ENDIF() diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py new file mode 100644 index 00000000..d2ec4d52 --- /dev/null +++ b/surround360_render/scripts/scanner_process_video.py @@ -0,0 +1,264 @@ +from scannerpy import Database, DeviceType +from scannerpy.stdlib import NetDescriptor, parsers, bboxes +import numpy as np +import argparse +import os +import signal +import subprocess +import sys +import struct +import scipy.misc + +from os import listdir, system +from os.path import isfile, join +from timeit import default_timer as timer + +current_process = None +def signal_term_handler(signal, frame): + if current_process: + print "Terminating process: " + current_process.name + "..." + current_process.terminate() + sys.exit(0) + +RENDER_COMMAND_TEMPLATE = """ +{SURROUND360_RENDER_DIR}/bin/TestRenderStereoPanorama +--logbuflevel -1 +--log_dir {LOG_DIR} +--stderrthreshold 0 +--v {VERBOSE_LEVEL} +--rig_json_file {RIG_JSON_FILE} +--imgs_dir {SRC_DIR}/rgb +--frame_number {FRAME_ID} +--output_data_dir {SRC_DIR} +--prev_frame_data_dir {PREV_FRAME_DIR} +--output_cubemap_path {OUT_CUBE_DIR}/cube_{FRAME_ID}.png +--output_equirect_path {OUT_EQR_DIR}/eqr_{FRAME_ID}.png +--cubemap_format {CUBEMAP_FORMAT} +--side_flow_alg {SIDE_FLOW_ALGORITHM} +--polar_flow_alg {POLAR_FLOW_ALGORITHM} +--poleremoval_flow_alg {POLEREMOVAL_FLOW_ALGORITHM} +--cubemap_width {CUBEMAP_WIDTH} +--cubemap_height {CUBEMAP_HEIGHT} +--eqr_width {EQR_WIDTH} +--eqr_height {EQR_HEIGHT} +--final_eqr_width {FINAL_EQR_WIDTH} +--final_eqr_height {FINAL_EQR_HEIGHT} +--interpupilary_dist 6.4 +--zero_parallax_dist 10000 +--sharpenning {SHARPENNING} +{EXTRA_FLAGS} +""" + +def start_subprocess(name, cmd): + global current_process + current_process = subprocess.Popen(cmd, shell=True) + current_process.name = name + current_process.communicate() + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, signal_term_handler) + + parser = argparse.ArgumentParser(description='batch process video frames') + parser.add_argument('--root_dir', help='path to frame container dir', required=True) + parser.add_argument('--surround360_render_dir', help='project root path, containing bin and scripts dirs', required=False, default='.') + parser.add_argument('--start_frame', help='first frame index', required=True) + parser.add_argument('--end_frame', help='last frame index', required=True) + parser.add_argument('--quality', help='3k,4k,6k,8k', required=True) + parser.add_argument('--cubemap_width', help='default is to not generate cubemaps', required=False, default=0) + parser.add_argument('--cubemap_height', help='default is to not generate cubemaps', required=False, default=0) + parser.add_argument('--cubemap_format', help='photo,video', required=False, default='photo') + parser.add_argument('--save_debug_images', dest='save_debug_images', action='store_true') + parser.add_argument('--enable_top', dest='enable_top', action='store_true') + parser.add_argument('--enable_bottom', dest='enable_bottom', action='store_true') + parser.add_argument('--enable_pole_removal', dest='enable_pole_removal', action='store_true') + parser.add_argument('--resume', dest='resume', action='store_true', help='looks for a previous frame optical flow instead of starting fresh') + parser.add_argument('--rig_json_file', help='path to rig json file', required=True) + parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True) + parser.add_argument('--verbose', dest='verbose', action='store_true') + parser.set_defaults(save_debug_images=False) + parser.set_defaults(enable_top=False) + parser.set_defaults(enable_bottom=False) + parser.set_defaults(enable_pole_removal=False) + args = vars(parser.parse_args()) + + root_dir = args["root_dir"] + surround360_render_dir = args["surround360_render_dir"] + log_dir = root_dir + "/logs" + out_eqr_frames_dir = root_dir + "/eqr_frames" + out_cube_frames_dir = root_dir + "/cube_frames" + flow_dir = root_dir + "/flow" + debug_dir = root_dir + "/debug" + min_frame = int(args["start_frame"]) + max_frame = int(args["end_frame"]) + cubemap_width = int(args["cubemap_width"]) + cubemap_height = int(args["cubemap_height"]) + cubemap_format = args["cubemap_format"] + quality = args["quality"] + save_debug_images = args["save_debug_images"] + enable_top = args["enable_top"] + enable_bottom = args["enable_bottom"] + enable_pole_removal = args["enable_pole_removal"] + resume = args["resume"] + rig_json_file = args["rig_json_file"] + flow_alg = args["flow_alg"] + verbose = args["verbose"] + + start_time = timer() + + frame_range = range(min_frame, max_frame + 1) + + render_params = { + "SURROUND360_RENDER_DIR": surround360_render_dir, + "LOG_DIR": log_dir, + "SRC_DIR": root_dir, + "PREV_FRAME_DIR": "NONE", + "OUT_EQR_DIR": out_eqr_frames_dir, + "OUT_CUBE_DIR": out_cube_frames_dir, + "CUBEMAP_WIDTH": cubemap_width, + "CUBEMAP_HEIGHT": cubemap_height, + "CUBEMAP_FORMAT": cubemap_format, + "RIG_JSON_FILE": rig_json_file, + "SIDE_FLOW_ALGORITHM": flow_alg, + "POLAR_FLOW_ALGORITHM": flow_alg, + "POLEREMOVAL_FLOW_ALGORITHM": flow_alg, + "EXTRA_FLAGS": "", + } + + + if save_debug_images: + render_params["EXTRA_FLAGS"] += " --save_debug_images" + + if enable_top: + render_params["EXTRA_FLAGS"] += " --enable_top" + + if enable_pole_removal and enable_bottom is False: + sys.stderr.write("Cannot use enable_pole_removal if enable_bottom is not used") + exit(1) + + if enable_bottom: + render_params["EXTRA_FLAGS"] += " --enable_bottom" + if enable_pole_removal: + render_params["EXTRA_FLAGS"] += " --enable_pole_removal" + render_params["EXTRA_FLAGS"] += " --bottom_pole_masks_dir " + root_dir + "/pole_masks" + + if quality == "3k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 3080 + render_params["EQR_HEIGHT"] = 1540 + render_params["FINAL_EQR_WIDTH"] = 3080 + render_params["FINAL_EQR_HEIGHT"] = 3080 + elif quality == "4k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 4200 + render_params["EQR_HEIGHT"] = 1024 + render_params["FINAL_EQR_WIDTH"] = 4096 + render_params["FINAL_EQR_HEIGHT"] = 2048 + elif quality == "6k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 6300 + render_params["EQR_HEIGHT"] = 3072 + render_params["FINAL_EQR_WIDTH"] = 6144 + render_params["FINAL_EQR_HEIGHT"] = 6144 + elif quality == "8k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 8400 + render_params["EQR_HEIGHT"] = 4096 + render_params["FINAL_EQR_WIDTH"] = 8192 + render_params["FINAL_EQR_HEIGHT"] = 8192 + else: + sys.stderr.write("Unrecognized quality setting: " + quality) + exit(1) + + collection_name = 'surround360' + idx_collection_name = 'surround360_index' + with Database() as db: + db.load_op('build/lib/libsurround360kernels.so', + 'build/source/scanner_kernels/surround360_pb2.py') + + if False or not db.has_collection(collection_name): + print "----------- [Render] loading surround360 collection" + sys.stdout.flush() + paths = [os.path.join(root_dir, 'rgb', 'cam{:d}'.format(i), 'vid.mp4') + for i in range(1, 15)] + collection, _ = db.ingest_video_collection(collection_name, paths, + force=True) + + idx_table_names = [] + num_rows = collection.tables(0).num_rows() + columns = ['camera_index'] + for c in range(0, len(paths)): + rows = [] + for i in range(num_rows): + rows.append([struct.pack('i', c)]) + table_name = 'surround_cam_idx_{:d}'.format(c) + db.new_table(table_name, columns, rows, force=True) + idx_table_names.append(table_name) + db.new_collection(idx_collection_name, idx_table_names, + force=True) + + + videos = db.collection(collection_name) + videos_idx = db.collection(idx_collection_name) + + print "----------- [Render] processing frames ", min_frame, max_frame + sys.stdout.flush() + if verbose: + print(render_params) + sys.stdout.flush() + + + in_columns = ["index", "frame", "frame_info", "camera_index"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) + + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for i, (vid, vid_idx) in enumerate(zip(videos.tables(), + videos_idx.tables())): + task = db.protobufs.Task() + task.output_table_name = vid.name() + str(i) + + sample = task.samples.add() + sample.table_name = vid.name() + sample.column_names.extend([c.name() for c in vid.columns()]) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + print(task) + + out_col = db.run(tasks, op, 'surround360_spherical', force=True) + t1 = out_col.tables(0) + for fi, tup in t1.load(['projected_frame', 'frame_info']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( + frame_info.height, + frame_info.width, + 4) + scipy.misc.toimage(frame[:,:,0:3]).save('test.png') + + + end_time = timer() + + if verbose: + total_runtime = end_time - start_time + avg_runtime = total_runtime / float(max_frame - min_frame + 1) + print "Render total runtime:", total_runtime, "sec" + print "Average runtime:", avg_runtime, "sec/frame" + sys.stdout.flush() diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp new file mode 100644 index 00000000..3cbc44ff --- /dev/null +++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp @@ -0,0 +1,146 @@ +#include "render/RigDescription.h" +#include "render/ImageWarper.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +#include + +using namespace scanner; + +namespace surround360 { + +namespace { +// measured in radians from forward +float approximateFov(const Camera& camera, const bool vertical) { + Camera::Vector2 a = camera.principal; + Camera::Vector2 b = camera.principal; + if (vertical) { + a.y() = 0; + b.y() = camera.resolution.y(); + } else { + a.x() = 0; + b.x() = camera.resolution.x(); + } + return acos(max( + camera.rig(a).direction().dot(camera.forward()), + camera.rig(b).direction().dot(camera.forward()))); +} + +// measured in radians from forward +float approximateFov(const Camera::Rig& rig, const bool vertical) { + float result = 0; + for (const auto& camera : rig) { + result = std::max(result, approximateFov(camera, vertical)); + } + return result; +} +} + +class ProjectSphericalKernelCPU : public VideoKernel { + public: + ProjectSphericalKernelCPU(const Kernel::Config& config) + : VideoKernel(config), + device_(config.devices[0]), + work_item_size_(config.work_item_size) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + // Initialize camera rig + rig_.reset(new RigDescription(args_.camera_rig_path())); + + hRadians_ = 2 * approximateFov(rig_->rigSideOnly, false); + vRadians_ = 2 * approximateFov(rig_->rigSideOnly, true); + } + + void reset() override { + is_reset_ = true; + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& frame_col = input_columns[0]; + auto& frame_info_col = input_columns[1]; + auto& camera_id_col = input_columns[2]; + check_frame_info(device_, frame_info_col); + + if (is_reset_) { + // Use the new camera id to update the spherical projection parameters + is_reset_ = false; + + camIdx_ = *((int*)camera_id_col.rows[0].buffer); + const Camera& camera = rig_->rigSideOnly[camIdx_]; + + // the negative sign here is so the camera array goes clockwise + const int numCameras = 14; + float direction = -float(camIdx_) / float(numCameras) * 2.0f * M_PI; + leftAngle_ = direction + hRadians_ / 2; + rightAngle_ = direction - hRadians_ / 2; + topAngle_ = vRadians_ / 2; + bottomAngle_ = -vRadians_ / 2; + } + + i32 input_count = (i32)frame_col.rows.size(); + size_t output_image_width = args_.eqr_width() * hRadians_ / (2 * M_PI); + size_t output_image_height = args_.eqr_height() * vRadians_ / M_PI; + size_t output_image_size = output_image_width * output_image_height * 4; + u8 *output_block = + new_block_buffer(device_, output_image_size * input_count, input_count); + + // Output frame info + scanner::proto::FrameInfo output_frame_info; + output_frame_info.set_width(output_image_width); + output_frame_info.set_height(output_image_height); + output_frame_info.set_channels(4); + u8 *output_frame_info_buffer = + new_block_buffer(device_, output_frame_info.ByteSize(), input_count); + output_frame_info.SerializeToArray(output_frame_info_buffer, + output_frame_info.ByteSize()); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat input(frame_info_.height(), frame_info_.width(), CV_8UC3, + input_columns[0].rows[i].buffer); + cv::Mat tmp; + cv::cvtColor(input, tmp, CV_BGR2BGRA); + + cv::Mat projection_image(output_image_height, output_image_width, CV_8UC4, + output_block + i * output_image_size); + + surround360::warper::bicubicRemapToSpherical( + projection_image, tmp, rig_->rigSideOnly[camIdx_], leftAngle_, + rightAngle_, topAngle_, bottomAngle_); + + output_columns[0].rows.push_back( + Row{projection_image.data, output_image_size}); + output_columns[1].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + } + } + + private: + surround360::proto::ProjectSphericalArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + i32 work_item_size_; + bool is_reset_ = true; + + float hRadians_; + float vRadians_; + + int camIdx_; + float leftAngle_; + float rightAngle_; + float topAngle_; + float bottomAngle_; +}; + +REGISTER_OP(ProjectSpherical) + .inputs({"frame", "frame_info", "camera_id"}) + .outputs({"projected_frame", "frame_info"}); + +REGISTER_KERNEL(ProjectSpherical, ProjectSphericalKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto new file mode 100644 index 00000000..f1c8aa5e --- /dev/null +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package surround360.proto; + +message ProjectSphericalArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + string camera_rig_path = 3; +} From 9f09264707d370799c101dcfbc45aa852de3bbb6 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sat, 8 Apr 2017 14:15:40 -0400 Subject: [PATCH 05/19] Add temporal optical flow kernel --- surround360_render/CMakeLists.txt | 5 +- .../source/render/RigDescription.h | 25 ++++ .../project_spherical_kernel_cpu.cpp | 27 ---- .../source/scanner_kernels/surround360.proto | 5 + .../temporal_optical_flow_kernel_cpu.cpp | 129 ++++++++++++++++++ .../source/test/TestRenderStereoPanorama.cpp | 25 ---- 6 files changed, 163 insertions(+), 53 deletions(-) create mode 100644 surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index a09618d2..c87aef72 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -528,7 +528,10 @@ IF (USE_SCANNER) include(${SCANNER_PATH}/cmake/Util/Op.cmake) build_op( LIB_NAME surround360kernels - CPP_SRCS source/scanner_kernels/project_spherical_kernel_cpu.cpp + CPP_SRCS + source/scanner_kernels/project_spherical_kernel_cpu.cpp + source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp + PROTO_SRC source/scanner_kernels/surround360.proto) target_link_libraries(surround360kernels PUBLIC LibVrCamera diff --git a/surround360_render/source/render/RigDescription.h b/surround360_render/source/render/RigDescription.h index 0d5ab739..ca32a3ff 100644 --- a/surround360_render/source/render/RigDescription.h +++ b/surround360_render/source/render/RigDescription.h @@ -62,4 +62,29 @@ struct RigDescription { } }; +// measured in radians from forward +inline float approximateFov(const Camera& camera, const bool vertical) { + Camera::Vector2 a = camera.principal; + Camera::Vector2 b = camera.principal; + if (vertical) { + a.y() = 0; + b.y() = camera.resolution.y(); + } else { + a.x() = 0; + b.x() = camera.resolution.x(); + } + return acos(max( + camera.rig(a).direction().dot(camera.forward()), + camera.rig(b).direction().dot(camera.forward()))); +} + +// measured in radians from forward +inline float approximateFov(const Camera::Rig& rig, const bool vertical) { + float result = 0; + for (const auto& camera : rig) { + result = std::max(result, approximateFov(camera, vertical)); + } + return result; +} + } // namespace surround360 diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp index 3cbc44ff..24b2d197 100644 --- a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp @@ -13,33 +13,6 @@ using namespace scanner; namespace surround360 { -namespace { -// measured in radians from forward -float approximateFov(const Camera& camera, const bool vertical) { - Camera::Vector2 a = camera.principal; - Camera::Vector2 b = camera.principal; - if (vertical) { - a.y() = 0; - b.y() = camera.resolution.y(); - } else { - a.x() = 0; - b.x() = camera.resolution.x(); - } - return acos(max( - camera.rig(a).direction().dot(camera.forward()), - camera.rig(b).direction().dot(camera.forward()))); -} - -// measured in radians from forward -float approximateFov(const Camera::Rig& rig, const bool vertical) { - float result = 0; - for (const auto& camera : rig) { - result = std::max(result, approximateFov(camera, vertical)); - } - return result; -} -} - class ProjectSphericalKernelCPU : public VideoKernel { public: ProjectSphericalKernelCPU(const Kernel::Config& config) diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto index f1c8aa5e..0d77a23f 100644 --- a/surround360_render/source/scanner_kernels/surround360.proto +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -7,3 +7,8 @@ message ProjectSphericalArgs { int32 eqr_width = 2; string camera_rig_path = 3; } + +message TemporalOpticalFlowArgs { + int32 overlap_image_width = 1; + string flow_algo = 2; +} diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp new file mode 100644 index 00000000..a3f7f073 --- /dev/null +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -0,0 +1,129 @@ +#include "render/NovelView.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { + +class TemporalOpticalFlowKernelCPU : public VideoKernel { + public: + TemporalOpticalFlowKernelCPU(const Kernel::Config& config) + : VideoKernel(config), + device_(config.devices[0]), + work_item_size_(config.work_item_size) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + overlap_image_width_ = args_.overlap_image_width(); + novel_view_gen_.reset( + new NovelViewGeneratorAsymmetricFlow(args_.flow_algo())); + } + + void reset() override { + prev_frame_flow_l_to_r_ = cv::Mat(); + prev_frame_flow_r_to_l_ = cv::Mat(); + prev_overlap_image_l_ = cv::Mat(); + prev_overlap_image_r_ = cv::Mat(); + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& left_frame_col = input_columns[0]; + auto& left_frame_info_col = input_columns[1]; + auto& right_frame_col = input_columns[2]; + auto& right_frame_info_col = input_columns[3]; + check_frame_info(device_, left_frame_info_col); + + i32 input_count = (i32)frame_col.rows.size(); + size_t output_image_width = frame_info_.width() - overlap_image_width_; + size_t output_image_height = frame_info_.height(); + size_t output_image_size = + output_image_width * output_image_height * 2 * sizeof(float); + u8 *output_block = new_block_buffer( + device_, output_image_size * input_count * 2, input_count * 2); + + // Output frame info + scanner::proto::FrameInfo output_frame_info; + output_frame_info.set_width(output_image_width); + output_frame_info.set_height(output_image_height); + output_frame_info.set_channels(2); + u8 *output_frame_info_buffer = new_block_buffer( + device_, output_frame_info.ByteSize(), input_count * 2); + output_frame_info.SerializeToArray(output_frame_info_buffer, + output_frame_info.ByteSize()); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC3, + input_columns[0].rows[i].buffer); + cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC3, + input_columns[0].rows[i].buffer); + + cv::Mat left_overlap_input = + left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, + overlap_image_width_, left_input.rows)); + cv::Mat right_overlap_input = + right_input(cv::Rect(0, 0, + overlap_image_width_, right_input.rows)); + + novel_view_gen_->prepare(left_overlap_input, right_overlap_input, + prev_frame_flow_l_to_r_, prev_frame_flow_r_to_l_, + prev_overlap_image_l_, prev_overlap_image_r_); + + prev_overlap_image_l_ = left_overlap_input; + prev_overlap_image_r_ = right_overlap_input; + prev_frame_flow_l_to_r_ = novel_view_gen_->getFlowLtoR(); + prev_frame_flow_r_to_l_ = novel_view_gen_->getFlowRtoL(); + + u8* left_output = output_block_buffer + output_image_size * (2 * i); + u8* right_output = output_block_buffer + output_image_size * (2 * i + 1); + auto& left_flow = novel_view_gen_->getFlowLtoR(); + auto& right_flow = novel_view_gen_->getFlowRtoL(); + for (i32 r = 0; r < left_flow.rows; ++r) { + memcpy(left_output + r * output_image_width * sizeof(float) * 2, + left_flow.data + left_flow.step * r, + left_flow.cols * sizeof(float) * 2); + memcpy(right_output + r * output_image_width * sizeof(float) * 2, + right_flow.data + right_flow.step * r, + right_flow.cols * sizeof(float) * 2); + } + + output_columns[0].rows.push_back( + Row{left_output, output_image_size}); + output_columns[1].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + output_columns[2].rows.push_back( + Row{right_output, output_image_size}); + output_columns[3].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + } + } + + private: + surround360::proto::TemporalOpticalFlowArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + i32 work_item_size_; + bool is_reset_ = true; + + std::unique_ptr novel_view_gen_; + int camIdx_; + cv::Mat prev_frame_flow_l_to_r_; + cv::Mat prev_frame_flow_r_to_l_; + cv::Mat prev_overlap_image_l_; + cv::Mat prev_overlap_image_r_; +}; + +REGISTER_OP(TemporalOpticalFlow) + .inputs({"projected_frame_left", "frame_info_left", "projeted_frame_right", + "frame_info_right"}) + .outputs({"flow_left", "frame_info_left", "flow_right", + "frame_info_right"}); + +REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/test/TestRenderStereoPanorama.cpp b/surround360_render/source/test/TestRenderStereoPanorama.cpp index 0f302feb..f6b5f669 100644 --- a/surround360_render/source/test/TestRenderStereoPanorama.cpp +++ b/surround360_render/source/test/TestRenderStereoPanorama.cpp @@ -71,31 +71,6 @@ DEFINE_string(cubemap_format, "video", "either video or photo const Camera::Vector3 kGlobalUp = Camera::Vector3::UnitZ(); -// measured in radians from forward -float approximateFov(const Camera& camera, const bool vertical) { - Camera::Vector2 a = camera.principal; - Camera::Vector2 b = camera.principal; - if (vertical) { - a.y() = 0; - b.y() = camera.resolution.y(); - } else { - a.x() = 0; - b.x() = camera.resolution.x(); - } - return acos(max( - camera.rig(a).direction().dot(camera.forward()), - camera.rig(b).direction().dot(camera.forward()))); -} - -// measured in radians from forward -float approximateFov(const Camera::Rig& rig, const bool vertical) { - float result = 0; - for (const auto& camera : rig) { - result = std::max(result, approximateFov(camera, vertical)); - } - return result; -} - void projectSideToSpherical( Mat& dst, const Mat& src, From 0eaac62c19e5315b481a666ce4fb8153e6a02c6b Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sat, 8 Apr 2017 16:45:32 -0400 Subject: [PATCH 06/19] Factor out project images in py file, make temporal kernel compute overlap --- .../scripts/scanner_process_video.py | 91 +++++++++++-------- .../source/scanner_kernels/surround360.proto | 4 +- .../temporal_optical_flow_kernel_cpu.cpp | 38 ++++++-- 3 files changed, 87 insertions(+), 46 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index d2ec4d52..b9fb9e85 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -55,6 +55,60 @@ def start_subprocess(name, cmd): current_process.name = name current_process.communicate() + +def project_images(db, videos, videos_idx, render_params): + in_columns = ["index", "frame", "frame_info", "camera_index"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) + + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for i, (vid, vid_idx) in enumerate(zip(videos.tables(), + videos_idx.tables())): + task = db.protobufs.Task() + task.output_table_name = vid.name() + str(i) + + sample = task.samples.add() + sample.table_name = vid.name() + sample.column_names.extend([c.name() for c in vid.columns()]) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + + return db.run(tasks, op, 'surround360_spherical', force=True) + + +def compute_temporal_flow(db, overlap, render_params): + in_columns = ["index", + "frame_left", "frame_info_left", + "frame_right", "frame_info_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.TemporalOpticalFlow() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], + args=args) + + # NOTE(apoms): WIP + pass + + if __name__ == "__main__": signal.signal(signal.SIGTERM, signal_term_handler) @@ -206,42 +260,7 @@ def start_subprocess(name, cmd): print(render_params) sys.stdout.flush() - - in_columns = ["index", "frame", "frame_info", "camera_index"] - input_op = db.ops.Input(in_columns) - - args = db.protobufs.ProjectSphericalArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) - - tasks = [] - sampler_args = db.protobufs.AllSamplerArgs() - sampler_args.warmup_size = 0 - sampler_args.sample_size = 32 - - for i, (vid, vid_idx) in enumerate(zip(videos.tables(), - videos_idx.tables())): - task = db.protobufs.Task() - task.output_table_name = vid.name() + str(i) - - sample = task.samples.add() - sample.table_name = vid.name() - sample.column_names.extend([c.name() for c in vid.columns()]) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - sample = task.samples.add() - sample.table_name = vid_idx.name() - sample.column_names.extend(['camera_index']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - tasks.append(task) - print(task) - - out_col = db.run(tasks, op, 'surround360_spherical', force=True) + out_col = project_images(db, videos, videos_idx) t1 = out_col.tables(0) for fi, tup in t1.load(['projected_frame', 'frame_info']): frame_info = db.protobufs.FrameInfo() diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto index 0d77a23f..aacb8aac 100644 --- a/surround360_render/source/scanner_kernels/surround360.proto +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -9,6 +9,6 @@ message ProjectSphericalArgs { } message TemporalOpticalFlowArgs { - int32 overlap_image_width = 1; - string flow_algo = 2; + string flow_algo = 1; + string camera_rig_path = 2; } diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index a3f7f073..cc2f99ac 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -1,4 +1,6 @@ -#include "render/NovelView.h" +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" #include "source/scanner_kernels/surround360.pb.h" #include "scanner/api/kernel.h" @@ -10,6 +12,9 @@ using namespace scanner; namespace surround360 { +using namespace optical_flow; +using namespace math_util; + class TemporalOpticalFlowKernelCPU : public VideoKernel { public: TemporalOpticalFlowKernelCPU(const Kernel::Config& config) @@ -18,7 +23,9 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { work_item_size_(config.work_item_size) { args_.ParseFromArray(config.args.data(), config.args.size()); - overlap_image_width_ = args_.overlap_image_width(); + rig_.reset(new RigDescription(args_.camera_rig_path())); + + overlap_image_width_ = -1; novel_view_gen_.reset( new NovelViewGeneratorAsymmetricFlow(args_.flow_algo())); } @@ -30,6 +37,20 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { prev_overlap_image_r_ = cv::Mat(); } + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + overlap_image_width_ = + float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees); + } + void execute(const BatchedColumns& input_columns, BatchedColumns& output_columns) override { auto& left_frame_col = input_columns[0]; @@ -37,13 +58,14 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { auto& right_frame_col = input_columns[2]; auto& right_frame_info_col = input_columns[3]; check_frame_info(device_, left_frame_info_col); + assert(overlap_image_width_ != -1); - i32 input_count = (i32)frame_col.rows.size(); + i32 input_count = (i32)left_frame_col.rows.size(); size_t output_image_width = frame_info_.width() - overlap_image_width_; size_t output_image_height = frame_info_.height(); size_t output_image_size = output_image_width * output_image_height * 2 * sizeof(float); - u8 *output_block = new_block_buffer( + u8 *output_block_buffer = new_block_buffer( device_, output_image_size * input_count * 2, input_count * 2); // Output frame info @@ -80,8 +102,8 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { u8* left_output = output_block_buffer + output_image_size * (2 * i); u8* right_output = output_block_buffer + output_image_size * (2 * i + 1); - auto& left_flow = novel_view_gen_->getFlowLtoR(); - auto& right_flow = novel_view_gen_->getFlowRtoL(); + const auto& left_flow = novel_view_gen_->getFlowLtoR(); + const auto& right_flow = novel_view_gen_->getFlowRtoL(); for (i32 r = 0; r < left_flow.rows; ++r) { memcpy(left_output + r * output_image_width * sizeof(float) * 2, left_flow.data + left_flow.step * r, @@ -107,10 +129,9 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { std::unique_ptr rig_; DeviceHandle device_; i32 work_item_size_; - bool is_reset_ = true; + int overlap_image_width_; std::unique_ptr novel_view_gen_; - int camIdx_; cv::Mat prev_frame_flow_l_to_r_; cv::Mat prev_frame_flow_r_to_l_; cv::Mat prev_overlap_image_l_; @@ -127,3 +148,4 @@ REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) .device(DeviceType::CPU) .num_devices(1); } + From 858ad72f2ee6980136be5798782ca9a7a5e99cde Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 9 Apr 2017 01:14:09 -0400 Subject: [PATCH 07/19] Working temporal optical flow kernel (2 of 4) --- .../scripts/scanner_process_video.py | 49 ++++++++++++++++--- .../temporal_optical_flow_kernel_cpu.cpp | 7 ++- 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index b9fb9e85..639c2bc5 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -95,18 +95,41 @@ def project_images(db, videos, videos_idx, render_params): def compute_temporal_flow(db, overlap, render_params): in_columns = ["index", - "frame_left", "frame_info_left", - "frame_right", "frame_info_right"] + "projected_left", "frame_info_left", + "projected_right", "frame_info_right"] input_op = db.ops.Input(in_columns) - args = db.protobufs.TemporalOpticalFlow() + args = db.protobufs.TemporalOpticalFlowArgs() args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] args.camera_rig_path = render_params["RIG_JSON_FILE"] op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], args=args) - # NOTE(apoms): WIP - pass + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for left_cam_idx in range(0, 14): + right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 + task = db.protobufs.Task() + task.output_table_name = 'surround360_flow_{:d}'.format(left_cam_idx) + + sample = task.samples.add() + sample.table_name = overlap.tables(left_cam_idx).name() + sample.column_names.extend(['index', 'projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = overlap.tables(right_cam_idx).name() + sample.column_names.extend(['projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + + return db.run(tasks, op, 'surround360_flow', force=True) if __name__ == "__main__": @@ -260,8 +283,8 @@ def compute_temporal_flow(db, overlap, render_params): print(render_params) sys.stdout.flush() - out_col = project_images(db, videos, videos_idx) - t1 = out_col.tables(0) + proj_col = project_images(db, videos, videos_idx, render_params) + t1 = proj_col.tables(0) for fi, tup in t1.load(['projected_frame', 'frame_info']): frame_info = db.protobufs.FrameInfo() frame_info.ParseFromString(tup[1]) @@ -272,6 +295,18 @@ def compute_temporal_flow(db, overlap, render_params): 4) scipy.misc.toimage(frame[:,:,0:3]).save('test.png') + flow_col = compute_temporal_flow(db, proj_col, render_params) + t1 = flow_col.tables(0) + for fi, tup in t1.load(['flow_left', 'frame_info_left']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.float32).reshape( + frame_info.height, + frame_info.width, + 2) + scipy.misc.toimage(frame[:,:,0]).save('flow_test_horiz.png') + scipy.misc.toimage(frame[:,:,1]).save('flow_test_vert.png') end_time = timer() diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index cc2f99ac..5be0ecd2 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -61,7 +61,7 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { assert(overlap_image_width_ != -1); i32 input_count = (i32)left_frame_col.rows.size(); - size_t output_image_width = frame_info_.width() - overlap_image_width_; + size_t output_image_width = overlap_image_width_; size_t output_image_height = frame_info_.height(); size_t output_image_size = output_image_width * output_image_height * 2 * sizeof(float); @@ -79,9 +79,9 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { output_frame_info.ByteSize()); for (i32 i = 0; i < input_count; ++i) { - cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC3, + cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC4, input_columns[0].rows[i].buffer); - cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC3, + cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC4, input_columns[0].rows[i].buffer); cv::Mat left_overlap_input = @@ -148,4 +148,3 @@ REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) .device(DeviceType::CPU) .num_devices(1); } - From 20f6dcd9ffd213c847b405f694e0dc9c80cf3fd2 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 9 Apr 2017 01:14:22 -0400 Subject: [PATCH 08/19] Enable setting images on NovelView directly --- surround360_render/source/optical_flow/NovelView.h | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/surround360_render/source/optical_flow/NovelView.h b/surround360_render/source/optical_flow/NovelView.h index af5cc632..0eb435e6 100644 --- a/surround360_render/source/optical_flow/NovelView.h +++ b/surround360_render/source/optical_flow/NovelView.h @@ -133,6 +133,12 @@ class NovelViewGenerator { const LazyNovelViewBuffer& lazyBuffer) = 0; // for debugging + virtual Mat getImageL() { return Mat(); } + virtual Mat getImageR() { return Mat(); } + + virtual void setImageL(const Mat& image) = 0; + virtual void setImageR(const Mat& image) = 0; + virtual Mat getFlowLtoR() { return Mat(); } virtual Mat getFlowRtoL() { return Mat(); } @@ -165,6 +171,12 @@ class NovelViewGeneratorLazyFlow : public NovelViewGenerator { pair combineLazyNovelViews(const LazyNovelViewBuffer& lazyBuffer); + Mat getImageL() { return imageL; } + Mat getImageR() { return imageR; } + + void setImageL(const Mat& image) { imageL = image; } + void setImageR(const Mat& image) { imageR = image; } + Mat getFlowLtoR() { return flowLtoR; } Mat getFlowRtoL() { return flowRtoL; } From 9951e567961dd2e9f3caae4a840164413505d5aa Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 9 Apr 2017 13:11:29 -0400 Subject: [PATCH 09/19] Add RenderStereoPanoramaChunk kernel (3 of 4) Fully working output panorama now --- surround360_render/CMakeLists.txt | 1 + .../scripts/scanner_process_video.py | 119 ++++++++++-- ...ender_stereo_panorama_chunk_kernel_cpu.cpp | 182 ++++++++++++++++++ .../source/scanner_kernels/surround360.proto | 9 + .../temporal_optical_flow_kernel_cpu.cpp | 4 +- 5 files changed, 292 insertions(+), 23 deletions(-) create mode 100644 surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index c87aef72..00247384 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -531,6 +531,7 @@ IF (USE_SCANNER) CPP_SRCS source/scanner_kernels/project_spherical_kernel_cpu.cpp source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp + source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp PROTO_SRC source/scanner_kernels/surround360.proto) target_link_libraries(surround360kernels PUBLIC diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index 639c2bc5..d983db5d 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -132,6 +132,63 @@ def compute_temporal_flow(db, overlap, render_params): return db.run(tasks, op, 'surround360_flow', force=True) +def render_stereo_panorama_chunks(db, overlap, flow, render_params): + in_columns = ["index", + "projected_left", "frame_info_left", + "flow_left", "flow_info_left", + "projected_right", "frame_info_right", + "flow_right", "flow_info_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.RenderStereoPanoramaChunkArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + op = db.ops.RenderStereoPanoramaChunk(inputs=[(input_op, in_columns[1:])], + args=args) + + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for left_cam_idx in range(0, 14): + right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 + task = db.protobufs.Task() + task.output_table_name = 'surround360_chunk_{:d}'.format(left_cam_idx) + + sample = task.samples.add() + sample.table_name = overlap.tables(left_cam_idx).name() + sample.column_names.extend(['index', 'projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = flow.tables(left_cam_idx).name() + sample.column_names.extend(['flow_left', 'frame_info_left']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = overlap.tables(right_cam_idx).name() + sample.column_names.extend(['projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = flow.tables(left_cam_idx).name() + sample.column_names.extend(['flow_right', 'frame_info_right']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + + return db.run(tasks, op, 'surround360_chunk', force=True) + + if __name__ == "__main__": signal.signal(signal.SIGTERM, signal_term_handler) @@ -284,29 +341,49 @@ def compute_temporal_flow(db, overlap, render_params): sys.stdout.flush() proj_col = project_images(db, videos, videos_idx, render_params) - t1 = proj_col.tables(0) - for fi, tup in t1.load(['projected_frame', 'frame_info']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - - frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( - frame_info.height, - frame_info.width, - 4) - scipy.misc.toimage(frame[:,:,0:3]).save('test.png') + if save_debug_images: + t1 = proj_col.tables(0) + for fi, tup in t1.load(['projected_frame', 'frame_info']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( + frame_info.height, + frame_info.width, + 4) + scipy.misc.toimage(frame[:,:,0:3]).save('test.png') flow_col = compute_temporal_flow(db, proj_col, render_params) - t1 = flow_col.tables(0) - for fi, tup in t1.load(['flow_left', 'frame_info_left']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - - frame = np.frombuffer(tup[0], dtype=np.float32).reshape( - frame_info.height, - frame_info.width, - 2) - scipy.misc.toimage(frame[:,:,0]).save('flow_test_horiz.png') - scipy.misc.toimage(frame[:,:,1]).save('flow_test_vert.png') + if save_debug_images: + t1 = flow_col.tables(0) + for fi, tup in t1.load(['flow_left', 'frame_info_left']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.float32).reshape( + frame_info.height, + frame_info.width, + 2) + frame = np.append(frame, np.zeros((frame_info.height, frame_info.width, 1)), axis=2) + scipy.misc.toimage(frame[:,:,:]).save('flow_test.png') + + chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) + if True: + panos = [None for _ in range(min_frame, max_frame + 1)] + for table in chunk_col.tables(): + for fi, tup in table.load(['chunk_left', 'frame_info_left']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( + frame_info.height, + frame_info.width, + 4) + if panos[fi] is None: + panos[fi] = frame + else: + panos[fi] = np.hstack((panos[fi], frame)) + scipy.misc.toimage(panos[1][:,:,0:3]).save('pano_test.png') end_time = timer() diff --git a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp new file mode 100644 index 00000000..ca114e98 --- /dev/null +++ b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp @@ -0,0 +1,182 @@ +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { + +using namespace optical_flow; +using namespace math_util; + +class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { + public: + RenderStereoPanoramaChunkKernelCPU(const Kernel::Config& config) + : VideoKernel(config), + device_(config.devices[0]), + work_item_size_(config.work_item_size) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + rig_.reset(new RigDescription(args_.camera_rig_path())); + + overlap_image_width_ = -1; + novel_view_gen_.reset( + new NovelViewGeneratorAsymmetricFlow(args_.flow_algo())); + } + + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + fov_horizontal_radians_ = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + overlap_image_width_ = float(camImageWidth) * + (overlapAngleDegrees / camFovHorizontalDegrees); + num_novel_views_ = + camImageWidth - overlap_image_width_; // per image pair + + const float v = + atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f)); + const float psi = + asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius); + const float vergeAtInfinitySlabDisplacement = + psi * (float(camImageWidth) / fov_horizontal_radians_); + const float theta = -M_PI / 2.0f + v + psi; + const float zeroParallaxNovelViewShiftPixels = + float(args_.eqr_width()) * (theta / (2.0f * M_PI)); + + int currChunkX = 0; + lazy_view_buffer_.reset(new LazyNovelViewBuffer(args_.eqr_width() / numCams, + camImageHeight)); + for (int nvIdx = 0; nvIdx < num_novel_views_; ++nvIdx) { + const float shift = float(nvIdx) / float(num_novel_views_); + const float slabShift = + float(camImageWidth) * 0.5f - float(num_novel_views_ - nvIdx); + + for (int v = 0; v < camImageHeight; ++v) { + lazy_view_buffer_->warpL[currChunkX][v] = + Point3f(slabShift + vergeAtInfinitySlabDisplacement, v, shift); + lazy_view_buffer_->warpR[currChunkX][v] = + Point3f(slabShift - vergeAtInfinitySlabDisplacement, v, shift); + } + ++currChunkX; + } + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& left_frame_col = input_columns[0]; + auto& left_frame_info_col = input_columns[1]; + auto& left_flow_col = input_columns[2]; + auto& left_flow_info_col = input_columns[3]; + + auto& right_frame_col = input_columns[4]; + auto& right_frame_info_col = input_columns[5]; + auto& right_flow_col = input_columns[6]; + auto& right_flow_info_col = input_columns[7]; + check_frame_info(device_, left_frame_info_col); + assert(overlap_image_width_ != -1); + + i32 input_count = (i32)left_frame_col.rows.size(); + size_t output_image_width = num_novel_views_; + size_t output_image_height = frame_info_.height(); + size_t output_image_size = + output_image_width * output_image_height * 4; + u8 *output_block_buffer = new_block_buffer( + device_, output_image_size * input_count * 2, input_count * 2); + + // Output frame info + scanner::proto::FrameInfo output_frame_info; + output_frame_info.set_width(output_image_width); + output_frame_info.set_height(output_image_height); + output_frame_info.set_channels(3); + u8 *output_frame_info_buffer = new_block_buffer( + device_, output_frame_info.ByteSize(), input_count * 2); + output_frame_info.SerializeToArray(output_frame_info_buffer, + output_frame_info.ByteSize()); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC4, + left_frame_col.rows[i].buffer); + cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC4, + right_frame_col.rows[i].buffer); + cv::Mat left_overlap_input = + left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, + overlap_image_width_, left_input.rows)); + cv::Mat right_overlap_input = + right_input(cv::Rect(0, 0, + overlap_image_width_, right_input.rows)); + + + cv::Mat left_flow(frame_info_.height(), overlap_image_width_, CV_32FC2, + left_flow_col.rows[i].buffer); + cv::Mat right_flow(frame_info_.height(), overlap_image_width_, CV_32FC2, + right_flow_col.rows[i].buffer); + + // Initialize NovelViewGenerator with images and flow since we are + // bypassing the prepare method + novel_view_gen_->setImageL(left_overlap_input); + novel_view_gen_->setImageR(right_overlap_input); + novel_view_gen_->setFlowLtoR(left_flow); + novel_view_gen_->setFlowRtoL(right_flow); + std::pair lazyNovelChunksLR = + novel_view_gen_->combineLazyNovelViews(*lazy_view_buffer_.get()); + const cv::Mat& chunkL = lazyNovelChunksLR.first; + const cv::Mat& chunkR = lazyNovelChunksLR.second; + + u8* left_output = output_block_buffer + output_image_size * (2 * i); + u8* right_output = output_block_buffer + output_image_size * (2 * i + 1); + for (i32 r = 0; r < chunkL.rows; ++r) { + memcpy(left_output + r * output_image_width * sizeof(char) * 4, + chunkL.data + chunkL.step * r, + chunkL.cols * sizeof(char) * 4); + memcpy(right_output + r * output_image_width * sizeof(char) * 4, + chunkR.data + chunkR.step * r, + chunkR.cols * sizeof(char) * 4); + } + + output_columns[0].rows.push_back( + Row{left_output, output_image_size}); + output_columns[1].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + output_columns[2].rows.push_back( + Row{right_output, output_image_size}); + output_columns[3].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + } + } + + private: + surround360::proto::RenderStereoPanoramaChunkArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + int work_item_size_; + float fov_horizontal_radians_; + int overlap_image_width_; + int num_novel_views_; + + std::unique_ptr novel_view_gen_; + std::unique_ptr lazy_view_buffer_; +}; + +REGISTER_OP(RenderStereoPanoramaChunk) + .inputs({"projected_frame_left", "frame_info_left", "flow_left", + "flow_info_left", "projected_framed_right", "frame_info_right", + "flow_right", "flow_info_right"}) + .outputs({"chunk_left", "frame_info_left", "chunk_right", + "frame_info_right"}); + +REGISTER_KERNEL(RenderStereoPanoramaChunk, RenderStereoPanoramaChunkKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto index aacb8aac..6a8c91a5 100644 --- a/surround360_render/source/scanner_kernels/surround360.proto +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -12,3 +12,12 @@ message TemporalOpticalFlowArgs { string flow_algo = 1; string camera_rig_path = 2; } + +message RenderStereoPanoramaChunkArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + string camera_rig_path = 3; + string flow_algo = 4; + float zero_parallax_dist = 5; + float interpupilary_dist = 6; +} diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index 5be0ecd2..c4f6f7ed 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -80,9 +80,9 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { for (i32 i = 0; i < input_count; ++i) { cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - input_columns[0].rows[i].buffer); + left_frame_col.rows[i].buffer); cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - input_columns[0].rows[i].buffer); + right_frame_col.rows[i].buffer); cv::Mat left_overlap_input = left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, From bdd661de28ec85c44458fa9c1e796da06ab548bd Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 9 Apr 2017 13:26:32 -0400 Subject: [PATCH 10/19] Add flow and stereo pipeline --- .../scripts/scanner_process_video.py | 89 +++++++++++++++---- 1 file changed, 73 insertions(+), 16 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index d983db5d..a4962eef 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -189,6 +189,58 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): return db.run(tasks, op, 'surround360_chunk', force=True) +def fused_flow_and_stereo_chunk(db, overlap, render_params): + in_columns = ["index", + "projected_left", "frame_info_left", + "projected_right", "frame_info_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + temporal_op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], + args=args) + + args = db.protobufs.RenderStereoPanoramaChunkArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + op = db.ops.RenderStereoPanoramaChunk( + inputs=[(input_op, ["projected_left", "frame_info_left"]), + (temporal_op, ["flow_left", "frame_info_left"]), + (input_op, ["projected_right", "frame_info_right"]), + (temporal_op, ["flow_right", "frame_info_right"])], + args=args) + + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for left_cam_idx in range(0, 14): + right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 + task = db.protobufs.Task() + task.output_table_name = 'surround360_flow_{:d}'.format(left_cam_idx) + + sample = task.samples.add() + sample.table_name = overlap.tables(left_cam_idx).name() + sample.column_names.extend(['index', 'projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = overlap.tables(right_cam_idx).name() + sample.column_names.extend(['projected_frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + + return db.run(tasks, op, 'surround360_chunk_fused', force=True) + if __name__ == "__main__": signal.signal(signal.SIGTERM, signal_term_handler) @@ -353,22 +405,27 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): 4) scipy.misc.toimage(frame[:,:,0:3]).save('test.png') - flow_col = compute_temporal_flow(db, proj_col, render_params) - if save_debug_images: - t1 = flow_col.tables(0) - for fi, tup in t1.load(['flow_left', 'frame_info_left']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - - frame = np.frombuffer(tup[0], dtype=np.float32).reshape( - frame_info.height, - frame_info.width, - 2) - frame = np.append(frame, np.zeros((frame_info.height, frame_info.width, 1)), axis=2) - scipy.misc.toimage(frame[:,:,:]).save('flow_test.png') - - chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) - if True: + fused = False + if fused: + chunk_col = fused_flow_and_stereo_chunk(db, proj_col, render_params) + else: + flow_col = compute_temporal_flow(db, proj_col, render_params) + if save_debug_images: + t1 = flow_col.tables(0) + for fi, tup in t1.load(['flow_left', 'frame_info_left']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.float32).reshape( + frame_info.height, + frame_info.width, + 2) + frame = np.append(frame, np.zeros((frame_info.height, frame_info.width, 1)), axis=2) + scipy.misc.toimage(frame[:,:,:]).save('flow_test.png') + + chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) + render = True + if render: panos = [None for _ in range(min_frame, max_frame + 1)] for table in chunk_col.tables(): for fi, tup in table.load(['chunk_left', 'frame_info_left']): From 12f2aac87d0a16efae74a31de6668b0ef7843962 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 9 Apr 2017 23:33:59 -0400 Subject: [PATCH 11/19] Add concat chunks kernel (4 of 4), tried out several pipeline variants --- surround360_render/CMakeLists.txt | 1 + .../scripts/scanner_process_video.py | 283 +++++++++++++----- .../concat_panorama_chunks_kernel_cpu.cpp | 120 ++++++++ .../source/scanner_kernels/surround360.proto | 9 + 4 files changed, 337 insertions(+), 76 deletions(-) create mode 100644 surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index 00247384..21bf8d80 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -532,6 +532,7 @@ IF (USE_SCANNER) source/scanner_kernels/project_spherical_kernel_cpu.cpp source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp + source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp PROTO_SRC source/scanner_kernels/surround360.proto) target_link_libraries(surround360kernels PUBLIC diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index a4962eef..b7460433 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -8,6 +8,7 @@ import sys import struct import scipy.misc +import time from os import listdir, system from os.path import isfile, join @@ -56,16 +57,7 @@ def start_subprocess(name, cmd): current_process.communicate() -def project_images(db, videos, videos_idx, render_params): - in_columns = ["index", "frame", "frame_info", "camera_index"] - input_op = db.ops.Input(in_columns) - - args = db.protobufs.ProjectSphericalArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) - +def project_tasks(db, videos, videos_idx): tasks = [] sampler_args = db.protobufs.AllSamplerArgs() sampler_args.warmup_size = 0 @@ -87,31 +79,81 @@ def project_images(db, videos, videos_idx, render_params): sample.column_names.extend(['camera_index']) sample.sampling_function = "All" sample.sampling_args = sampler_args.SerializeToString() - tasks.append(task) + return tasks - return db.run(tasks, op, 'surround360_spherical', force=True) +def project_tasks_join(db, videos, videos_idx): + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 -def compute_temporal_flow(db, overlap, render_params): - in_columns = ["index", - "projected_left", "frame_info_left", - "projected_right", "frame_info_right"] + for i in range(len(videos.tables())): + left_cam_idx = i + if i == len(videos.tables()) - 1: + right_cam_idx = 0 + else: + right_cam_idx = left_cam_idx + 1 + + vid_left = videos.tables(left_cam_idx) + vid_idx_left = videos_idx.tables(left_cam_idx) + vid_right = videos.tables(right_cam_idx) + vid_idx_right = videos_idx.tables(right_cam_idx) + + task = db.protobufs.Task() + task.output_table_name = vid_left.name() + str(i) + + sample = task.samples.add() + sample.table_name = vid_left.name() + sample.column_names.extend([c.name() for c in vid_left.columns()]) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx_left.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_right.name() + sample.column_names.extend(['frame', 'frame_info']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx_right.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + return tasks + + +def project_images(db, videos, videos_idx, render_params): + in_columns = ["index", "frame", "frame_info", "camera_index"] input_op = db.ops.Input(in_columns) - args = db.protobufs.TemporalOpticalFlowArgs() - args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] args.camera_rig_path = render_params["RIG_JSON_FILE"] - op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], - args=args) + op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) + tasks = project_tasks(db, videos, videos_idx) + return db.run(tasks, op, 'surround360_spherical', force=True) + + +def flow_tasks(db, overlap): tasks = [] sampler_args = db.protobufs.AllSamplerArgs() sampler_args.warmup_size = 0 sampler_args.sample_size = 32 for left_cam_idx in range(0, 14): - right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 + right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 0 task = db.protobufs.Task() task.output_table_name = 'surround360_flow_{:d}'.format(left_cam_idx) @@ -128,6 +170,21 @@ def compute_temporal_flow(db, overlap, render_params): sample.sampling_args = sampler_args.SerializeToString() tasks.append(task) + return tasks + + +def compute_temporal_flow(db, overlap, render_params): + in_columns = ["index", + "projected_left", "frame_info_left", + "projected_right", "frame_info_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], + args=args) + tasks = flow_tasks(db, overlap) return db.run(tasks, op, 'surround360_flow', force=True) @@ -156,7 +213,7 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): sampler_args.sample_size = 32 for left_cam_idx in range(0, 14): - right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 + right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 0 task = db.protobufs.Task() task.output_table_name = 'surround360_chunk_{:d}'.format(left_cam_idx) @@ -189,6 +246,94 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): return db.run(tasks, op, 'surround360_chunk', force=True) +def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): + num_cams = 14 + in_columns = ["index"] + for c in range(num_cams): + in_columns += ["chunk{:d}".format(c), + "frame_info{:d}".format(c)] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.ConcatPanoramaChunksArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = is_left + op = db.ops.ConcatPanoramaChunks(inputs=[(input_op, in_columns[1:])], + args=args) + + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + task = db.protobufs.Task() + task.output_table_name = 'surround360_pano' + sample = task.samples.add() + sample.table_name = chunks.tables(0).name() + sample.column_names.extend(['index', 'chunk_left', 'frame_info_left']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + for cam_idx in range(1, num_cams): + sample = task.samples.add() + sample.table_name = chunks.tables(cam_idx).name() + sample.column_names.extend(['chunk_left', 'frame_info_left']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + tasks.append(task) + + return db.run(tasks, op, 'surround360_pano', force=True) + + +def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params): + in_columns = ["index", + "frame_left", "frame_info_left", "camera_index_left", + "frame_right", "frame_info_right", "camera_index_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + proj_left_op = db.ops.ProjectSpherical( + inputs=[(input_op, + ["frame_left", "frame_info_left", "camera_index_left"])], + args=args) + proj_right_op = db.ops.ProjectSpherical( + inputs=[(input_op, + ["frame_right", "frame_info_right", "camera_index_right"])], + args=args) + + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + temporal_op = db.ops.TemporalOpticalFlow( + inputs=[ + (proj_left_op, ["projected_frame", "frame_info"]), + (proj_right_op, ["projected_frame", "frame_info"])], + args=args) + + args = db.protobufs.RenderStereoPanoramaChunkArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + op = db.ops.RenderStereoPanoramaChunk( + inputs=[(proj_left_op, ["projected_frame", "frame_info"]), + (temporal_op, ["flow_left", "frame_info_left"]), + (proj_right_op, ["projected_frame", "frame_info"]), + (temporal_op, ["flow_right", "frame_info_right"])], + args=args) + + tasks = project_tasks_join(db, videos, videos_idx) + + return db.run(tasks, op, 'surround360_chunk', force=True) + + def fused_flow_and_stereo_chunk(db, overlap, render_params): in_columns = ["index", "projected_left", "frame_info_left", @@ -215,32 +360,11 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): (temporal_op, ["flow_right", "frame_info_right"])], args=args) - tasks = [] - sampler_args = db.protobufs.AllSamplerArgs() - sampler_args.warmup_size = 0 - sampler_args.sample_size = 32 - - for left_cam_idx in range(0, 14): - right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 1 - task = db.protobufs.Task() - task.output_table_name = 'surround360_flow_{:d}'.format(left_cam_idx) - - sample = task.samples.add() - sample.table_name = overlap.tables(left_cam_idx).name() - sample.column_names.extend(['index', 'projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - sample = task.samples.add() - sample.table_name = overlap.tables(right_cam_idx).name() - sample.column_names.extend(['projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - tasks.append(task) + tasks = flow_tasks(db, overlap) return db.run(tasks, op, 'surround360_chunk_fused', force=True) + if __name__ == "__main__": signal.signal(signal.SIGTERM, signal_term_handler) @@ -357,7 +481,9 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): collection_name = 'surround360' idx_collection_name = 'surround360_index' + db_start = timer() with Database() as db: + print('DB', timer() - db_start) db.load_op('build/lib/libsurround360kernels.so', 'build/source/scanner_kernels/surround360_pb2.py') @@ -392,22 +518,31 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): print(render_params) sys.stdout.flush() - proj_col = project_images(db, videos, videos_idx, render_params) - if save_debug_images: - t1 = proj_col.tables(0) - for fi, tup in t1.load(['projected_frame', 'frame_info']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - - frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( - frame_info.height, - frame_info.width, - 4) - scipy.misc.toimage(frame[:,:,0:3]).save('test.png') - - fused = False - if fused: + fused_4 = False + fused_3 = True + fused_2 = False + if fused_4: + chunk_col = flow_chunk_pano(db, proj_col, render_params) + elif fused_3: + flow_stereo_start = timer() + chunk_col = fused_project_flow_and_stereo_chunk(db, videos, videos_idx, + render_params) + print('Proj flow stereo', timer() - flow_stereo_start) + concat_start = timer() + pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, + True) + print('Concat', timer() - concat_start) + elif fused_2: + proj_start = timer() + proj_col = project_images(db, videos, videos_idx, render_params) + print('Proj', timer() - proj_start) + flow_stereo_start = timer() chunk_col = fused_flow_and_stereo_chunk(db, proj_col, render_params) + print('Flow stereo', timer() - flow_stereo_start) + concat_start = timer() + pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, + True) + print('Concat', timer() - concat_start) else: flow_col = compute_temporal_flow(db, proj_col, render_params) if save_debug_images: @@ -424,23 +559,19 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): scipy.misc.toimage(frame[:,:,:]).save('flow_test.png') chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) - render = True - if render: - panos = [None for _ in range(min_frame, max_frame + 1)] - for table in chunk_col.tables(): - for fi, tup in table.load(['chunk_left', 'frame_info_left']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( - frame_info.height, - frame_info.width, - 4) - if panos[fi] is None: - panos[fi] = frame - else: - panos[fi] = np.hstack((panos[fi], frame)) - scipy.misc.toimage(panos[1][:,:,0:3]).save('pano_test.png') + png_start = timer() + left_table = pano_col.tables(0) + for fi, tup in left_table.load(['panorama', 'frame_info']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( + frame_info.height, + frame_info.width, + 3) + scipy.misc.toimage(frame[:,:,0:3]).save('pano_test.png') + print('To png', timer() - png_start) end_time = timer() diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp new file mode 100644 index 00000000..3a8e362b --- /dev/null +++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp @@ -0,0 +1,120 @@ +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { + +using namespace optical_flow; +using namespace math_util; + +class ConcatPanoramaChunksKernelCPU : public VideoKernel { + public: + ConcatPanoramaChunksKernelCPU(const Kernel::Config& config) + : VideoKernel(config), + device_(config.devices[0]), + work_item_size_(config.work_item_size) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + rig_.reset(new RigDescription(args_.camera_rig_path())); + + num_chunks_ = config.input_columns.size() / 2; + } + + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + const int overlapImageWidth = + float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees); + + const float v = + atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f)); + const float psi = + asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius); + const float vergeAtInfinitySlabDisplacement = + psi * (float(camImageWidth) / fovHorizontalRadians); + const float theta = -M_PI / 2.0f + v + psi; + zeroParallaxNovelViewShiftPixels_ = + float(args_.eqr_width()) * (theta / (2.0f * M_PI)); + if (!args_.left()) { + zeroParallaxNovelViewShiftPixels_ *= -1; + } + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + check_frame_info(device_, input_columns[1]); + + i32 input_count = (i32)input_columns[0].rows.size(); + size_t output_image_width = frame_info_.width() * num_chunks_; + size_t output_image_height = frame_info_.height(); + size_t output_image_size = + output_image_width * output_image_height * 3; + u8 *output_block_buffer = new_block_buffer( + device_, output_image_size * input_count, input_count); + + // Output frame info + scanner::proto::FrameInfo output_frame_info; + output_frame_info.set_width(output_image_width); + output_frame_info.set_height(output_image_height); + output_frame_info.set_channels(3); + u8 *output_frame_info_buffer = new_block_buffer( + device_, output_frame_info.ByteSize(), input_count); + output_frame_info.SerializeToArray(output_frame_info_buffer, + output_frame_info.ByteSize()); + + std::vector pano_chunks(num_chunks_, Mat()); + cv::Mat pano; + for (i32 i = 0; i < input_count; ++i) { + for (i32 c = 0; c < num_chunks_; ++c) { + auto &chunk_col = input_columns[c * 2]; + pano_chunks[c] = cv::Mat(frame_info_.height(), frame_info_.width(), + CV_8UC4, chunk_col.rows[i].buffer); + cv::cvtColor(pano_chunks[c], pano_chunks[c], CV_BGRA2BGR); + } + pano = stackHorizontal(pano_chunks); + pano = offsetHorizontalWrap(pano, zeroParallaxNovelViewShiftPixels_); + + u8 *output = output_block_buffer + output_image_size * i; + for (i32 r = 0; r < pano.rows; ++r) { + memcpy(output + r * output_image_width * sizeof(char) * 3, + pano.data + pano.step * r, pano.cols * sizeof(char) * 3); + } + + output_columns[0].rows.push_back(Row{output, output_image_size}); + output_columns[1].rows.push_back( + Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + } + } + + private: + surround360::proto::ConcatPanoramaChunksArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + int work_item_size_; + int num_chunks_; + float zeroParallaxNovelViewShiftPixels_; +}; + +REGISTER_OP(ConcatPanoramaChunks) + .variadic_inputs() + .outputs({"panorama", "frame_info"}); + +REGISTER_KERNEL(ConcatPanoramaChunks, ConcatPanoramaChunksKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto index 6a8c91a5..04652521 100644 --- a/surround360_render/source/scanner_kernels/surround360.proto +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -21,3 +21,12 @@ message RenderStereoPanoramaChunkArgs { float zero_parallax_dist = 5; float interpupilary_dist = 6; } + +message ConcatPanoramaChunksArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + string camera_rig_path = 3; + float zero_parallax_dist = 4; + float interpupilary_dist = 5; + bool left = 6; +} From 584ebca34775738e4a257a97d49bb49c5d089e44 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Mon, 24 Apr 2017 23:19:12 -0400 Subject: [PATCH 12/19] Update to new scanner API --- surround360_render/CMakeLists.txt | 2 +- .../scripts/batch_process_video.py | 1 + .../scripts/scanner_process_video.py | 384 ++++++++++-------- .../calibration/GeometricCalibration.cpp | 20 +- .../source/camera_isp/Raw2Rgb.cpp | 2 + .../concat_panorama_chunks_kernel_cpu.cpp | 37 +- .../project_spherical_kernel_cpu.cpp | 39 +- ...ender_stereo_panorama_chunk_kernel_cpu.cpp | 66 +-- .../temporal_optical_flow_kernel_cpu.cpp | 54 +-- 9 files changed, 290 insertions(+), 315 deletions(-) diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index 21bf8d80..d48ab7b0 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -57,7 +57,7 @@ ADD_CUSTOM_TARGET( _source SOURCES ${SRC} ) ### Halide support IF (DEFINED HALIDE_DIR) - INCLUDE("${HALIDE_DIR}/../HalideGenerator.cmake") + INCLUDE("${HALIDE_DIR}/HalideGenerator.cmake") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_HALIDE") diff --git a/surround360_render/scripts/batch_process_video.py b/surround360_render/scripts/batch_process_video.py index 6be7741e..7a171b66 100755 --- a/surround360_render/scripts/batch_process_video.py +++ b/surround360_render/scripts/batch_process_video.py @@ -192,6 +192,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l render_params["FINAL_EQR_HEIGHT"] = 6144 elif quality == "8k": render_params["SHARPENNING"] = 0.25 + render_params["SHARPENNING"] = 0.0 render_params["EQR_WIDTH"] = 8400 render_params["EQR_HEIGHT"] = 4096 render_params["FINAL_EQR_WIDTH"] = 8192 diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index b7460433..e5c0cf79 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -1,4 +1,4 @@ -from scannerpy import Database, DeviceType +from scannerpy import Database, DeviceType, Job from scannerpy.stdlib import NetDescriptor, parsers, bboxes import numpy as np import argparse @@ -118,7 +118,7 @@ def project_tasks_join(db, videos, videos_idx): sample = task.samples.add() sample.table_name = vid_right.name() - sample.column_names.extend(['frame', 'frame_info']) + sample.column_names.extend(['frame']) sample.sampling_function = "All" sample.sampling_args = sampler_args.SerializeToString() @@ -133,127 +133,90 @@ def project_tasks_join(db, videos, videos_idx): def project_images(db, videos, videos_idx, render_params): - in_columns = ["index", "frame", "frame_info", "camera_index"] - input_op = db.ops.Input(in_columns) + jobs = [] + for i in range(len(video.tables())): + left_frame = videos.tables(i).as_op().all() + left_cam_idx = videos_idx.tables(i).as_op().all() - args = db.protobufs.ProjectSphericalArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - op = db.ops.ProjectSpherical(inputs=[(input_op, in_columns[1:])], args=args) + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + proj_frame = db.ops.ProjectSpherical(frame = frame, camra_id = cam_idx, + args=args) - tasks = project_tasks(db, videos, videos_idx) - return db.run(tasks, op, 'surround360_spherical', force=True) - - -def flow_tasks(db, overlap): - tasks = [] - sampler_args = db.protobufs.AllSamplerArgs() - sampler_args.warmup_size = 0 - sampler_args.sample_size = 32 + job = Job(columns = [proj_frame], + name = 'surround360_spherical_{:d}'.format(i)) + jobs.append(job) - for left_cam_idx in range(0, 14): - right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 0 - task = db.protobufs.Task() - task.output_table_name = 'surround360_flow_{:d}'.format(left_cam_idx) - - sample = task.samples.add() - sample.table_name = overlap.tables(left_cam_idx).name() - sample.column_names.extend(['index', 'projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - sample = task.samples.add() - sample.table_name = overlap.tables(right_cam_idx).name() - sample.column_names.extend(['projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - tasks.append(task) - return tasks + return db.run(jobs, 'surround360_spherical', force=True) def compute_temporal_flow(db, overlap, render_params): - in_columns = ["index", - "projected_left", "frame_info_left", - "projected_right", "frame_info_right"] - input_op = db.ops.Input(in_columns) - - args = db.protobufs.TemporalOpticalFlowArgs() - args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], - args=args) - tasks = flow_tasks(db, overlap) + jobs = [] + for i in range(len(overlap.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(overlap.tables()) - return db.run(tasks, op, 'surround360_flow', force=True) + left_proj_frame = overlap.tables(left_cam_idx).as_op().all() + right_proj_frame = overlap.tables(right_cam_idx).as_op().all() + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + args=args) -def render_stereo_panorama_chunks(db, overlap, flow, render_params): - in_columns = ["index", - "projected_left", "frame_info_left", - "flow_left", "flow_info_left", - "projected_right", "frame_info_right", - "flow_right", "flow_info_right"] - input_op = db.ops.Input(in_columns) + job = Job(columns = [left_flow, right_flow], + name = 'surround360_flow_{:d}'.format(i)) + jobs.append(job) - args = db.protobufs.RenderStereoPanoramaChunkArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] - args.zero_parallax_dist = 10000 - args.interpupilary_dist = 6.4 - op = db.ops.RenderStereoPanoramaChunk(inputs=[(input_op, in_columns[1:])], - args=args) + return db.run(jobs, 'surround360_flow', force=True) - tasks = [] - sampler_args = db.protobufs.AllSamplerArgs() - sampler_args.warmup_size = 0 - sampler_args.sample_size = 32 - for left_cam_idx in range(0, 14): - right_cam_idx = left_cam_idx + 1 if left_cam_idx != 13 else 0 - task = db.protobufs.Task() - task.output_table_name = 'surround360_chunk_{:d}'.format(left_cam_idx) - - sample = task.samples.add() - sample.table_name = overlap.tables(left_cam_idx).name() - sample.column_names.extend(['index', 'projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - - sample = task.samples.add() - sample.table_name = flow.tables(left_cam_idx).name() - sample.column_names.extend(['flow_left', 'frame_info_left']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() +def render_stereo_panorama_chunks(db, overlap, flow, render_params): + jobs = [] + for i in range(len(video.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(video.tables()) - sample = task.samples.add() - sample.table_name = overlap.tables(right_cam_idx).name() - sample.column_names.extend(['projected_frame', 'frame_info']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() + left_proj_frame = overlap.tables(left_cam_idx).as_op().all() + left_flow = flow.tables(left_cam_idx).as_op().all() + right_proj_frame = videos.tables(right_cam_idx).as_op().all() + right_flow = flow_idx.tables(right_cam_idx).as_op().all() - sample = task.samples.add() - sample.table_name = flow.tables(left_cam_idx).name() - sample.column_names.extend(['flow_right', 'frame_info_right']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) - tasks.append(task) + job = Job(columns = [left_chunk, right_chunk], + name = 'surround360_chunk_{:d}'.format(i)) + jobs.append(job) - return db.run(tasks, op, 'surround360_chunk', force=True) + return db.run(jobs, force=True) def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): num_cams = 14 - in_columns = ["index"] + left_inputs = [] + right_inputs = [] + print(chunks) for c in range(num_cams): - in_columns += ["chunk{:d}".format(c), - "frame_info{:d}".format(c)] - input_op = db.ops.Input(in_columns) + left_chunk, right_chunk = chunks[c].as_op().all(item_size=50) + left_inputs.append(left_chunk) + right_inputs.append(right_chunk) + jobs = [] args = db.protobufs.ConcatPanoramaChunksArgs() args.eqr_width = render_params["EQR_WIDTH"] args.eqr_height = render_params["EQR_HEIGHT"] @@ -261,77 +224,144 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): args.zero_parallax_dist = 10000 args.interpupilary_dist = 6.4 args.left = is_left - op = db.ops.ConcatPanoramaChunks(inputs=[(input_op, in_columns[1:])], - args=args) + panorama_left = db.ops.ConcatPanoramaChunks(*left_inputs, args=args) - tasks = [] - sampler_args = db.protobufs.AllSamplerArgs() - sampler_args.warmup_size = 0 - sampler_args.sample_size = 32 - - task = db.protobufs.Task() - task.output_table_name = 'surround360_pano' - sample = task.samples.add() - sample.table_name = chunks.tables(0).name() - sample.column_names.extend(['index', 'chunk_left', 'frame_info_left']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - for cam_idx in range(1, num_cams): - sample = task.samples.add() - sample.table_name = chunks.tables(cam_idx).name() - sample.column_names.extend(['chunk_left', 'frame_info_left']) - sample.sampling_function = "All" - sample.sampling_args = sampler_args.SerializeToString() - tasks.append(task) + args = db.protobufs.ConcatPanoramaChunksArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = False + panorama_right = db.ops.ConcatPanoramaChunks(*right_inputs, args=args) + job = Job(columns = [panorama_left, panorama_right], + name = 'surround360_pano_both') - return db.run(tasks, op, 'surround360_pano', force=True) + return db.run(job, force=True, pipeline_instances_per_node=1) +def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, + start, end): + jobs = [] + item_size = 10 -def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params): - in_columns = ["index", - "frame_left", "frame_info_left", "camera_index_left", - "frame_right", "frame_info_right", "camera_index_right"] - input_op = db.ops.Input(in_columns) - - args = db.protobufs.ProjectSphericalArgs() + proj_frames = [] + for i in range(len(videos.tables())): + left_cam_idx = i + frame = videos.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) + cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + proj_frame = db.ops.ProjectSpherical( + frame = frame, + camera_id = cam_idx, + args=args) + proj_frames.append(proj_frame) + + left_chunks = [] + right_chunks = [] + for i in range(len(videos.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + + left_proj_frame = proj_frames[left_cam_idx] + right_proj_frame = proj_frames[right_cam_idx] + + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + camera_rig_path = render_params["RIG_JSON_FILE"]) + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + left_chunks.append(left_chunk) + right_chunks.append(right_chunk) + + jobs = [] + args = db.protobufs.ConcatPanoramaChunksArgs() args.eqr_width = render_params["EQR_WIDTH"] args.eqr_height = render_params["EQR_HEIGHT"] args.camera_rig_path = render_params["RIG_JSON_FILE"] - proj_left_op = db.ops.ProjectSpherical( - inputs=[(input_op, - ["frame_left", "frame_info_left", "camera_index_left"])], - args=args) - proj_right_op = db.ops.ProjectSpherical( - inputs=[(input_op, - ["frame_right", "frame_info_right", "camera_index_right"])], - args=args) - - args = db.protobufs.TemporalOpticalFlowArgs() - args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - temporal_op = db.ops.TemporalOpticalFlow( - inputs=[ - (proj_left_op, ["projected_frame", "frame_info"]), - (proj_right_op, ["projected_frame", "frame_info"])], - args=args) + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = True + panorama_left = db.ops.ConcatPanoramaChunks(*left_chunks, args=args) - args = db.protobufs.RenderStereoPanoramaChunkArgs() + args = db.protobufs.ConcatPanoramaChunksArgs() args.eqr_width = render_params["EQR_WIDTH"] args.eqr_height = render_params["EQR_HEIGHT"] args.camera_rig_path = render_params["RIG_JSON_FILE"] - args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] args.zero_parallax_dist = 10000 args.interpupilary_dist = 6.4 - op = db.ops.RenderStereoPanoramaChunk( - inputs=[(proj_left_op, ["projected_frame", "frame_info"]), - (temporal_op, ["flow_left", "frame_info_left"]), - (proj_right_op, ["projected_frame", "frame_info"]), - (temporal_op, ["flow_right", "frame_info_right"])], - args=args) + args.left = False + panorama_right = db.ops.ConcatPanoramaChunks(*right_chunks, args=args) - tasks = project_tasks_join(db, videos, videos_idx) + job = Job(columns = [panorama_left, panorama_right], + name = 'surround360_pano_both') + return db.run(job, pipeline_instances_per_node=1, + force=True) - return db.run(tasks, op, 'surround360_chunk', force=True) + +def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): + jobs = [] + item_size = 11 + for i in range(len(videos.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + + left_frame = videos.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) + left_cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) + right_frame = videos.tables(right_cam_idx).as_op().range(start, end, item_size = item_size) + right_cam_idx = videos_idx.tables(right_cam_idx).as_op().range(start, end, item_size = item_size) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + left_proj_frame = db.ops.ProjectSpherical( + frame = left_frame, + camera_id = left_cam_idx, + args=args) + right_proj_frame = db.ops.ProjectSpherical( + frame = right_frame, + camera_id = right_cam_idx, + args=args) + + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + camera_rig_path = render_params["RIG_JSON_FILE"]) + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + + job = Job(columns = [left_chunk.lossless(), right_chunk.lossless()], + name = 'surround360_chunk_{:d}'.format(i)) + jobs.append(job) + + return db.run(jobs, force=True,pipeline_instances_per_node=1) def fused_flow_and_stereo_chunk(db, overlap, render_params): @@ -487,7 +517,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): db.load_op('build/lib/libsurround360kernels.so', 'build/source/scanner_kernels/surround360_pb2.py') - if False or not db.has_collection(collection_name): + if not db.has_collection(collection_name): print "----------- [Render] loading surround360 collection" sys.stdout.flush() paths = [os.path.join(root_dir, 'rgb', 'cam{:d}'.format(i), 'vid.mp4') @@ -512,26 +542,37 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): videos = db.collection(collection_name) videos_idx = db.collection(idx_collection_name) + start_time = timer() print "----------- [Render] processing frames ", min_frame, max_frame sys.stdout.flush() if verbose: print(render_params) sys.stdout.flush() - fused_4 = False - fused_3 = True - fused_2 = False + visualize = False + fused_4 = True # Fused project, flow, stereo chunk, and pano + fused_3 = False # Fused project, flow, and stereo chunk + fused_2 = False # Fused flow and stereo chunk + fused_1 = False # Nothing fused if fused_4: - chunk_col = flow_chunk_pano(db, proj_col, render_params) + pano_col = fused_project_flow_chunk_concat(db, videos, videos_idx, + render_params, + min_frame, max_frame + 1) + pano_col.profiler().write_trace('fused4.trace') elif fused_3: flow_stereo_start = timer() chunk_col = fused_project_flow_and_stereo_chunk(db, videos, videos_idx, - render_params) + render_params, + min_frame, max_frame + 1) + print(db.summarize()) + chunk_col = [db.table('surround360_chunk_{:d}'.format(i)) + for i in range(14)] print('Proj flow stereo', timer() - flow_stereo_start) concat_start = timer() pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, True) print('Concat', timer() - concat_start) + chunk_col[0].profiler().write_trace('fused3.trace') elif fused_2: proj_start = timer() proj_col = project_images(db, videos, videos_idx, render_params) @@ -543,7 +584,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, True) print('Concat', timer() - concat_start) - else: + elif fused_1: flow_col = compute_temporal_flow(db, proj_col, render_params) if save_debug_images: t1 = flow_col.tables(0) @@ -561,16 +602,11 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) png_start = timer() - left_table = pano_col.tables(0) - for fi, tup in left_table.load(['panorama', 'frame_info']): - frame_info = db.protobufs.FrameInfo() - frame_info.ParseFromString(tup[1]) - - frame = np.frombuffer(tup[0], dtype=np.uint8).reshape( - frame_info.height, - frame_info.width, - 3) - scipy.misc.toimage(frame[:,:,0:3]).save('pano_test.png') + pano_col = db.table('surround360_pano_both') + # left_table = pano_col[0] + # right_table = pano_col[1] + if visualize: + left_table.columns('panorama').save_mp4('pano_test.mp4') print('To png', timer() - png_start) end_time = timer() diff --git a/surround360_render/source/calibration/GeometricCalibration.cpp b/surround360_render/source/calibration/GeometricCalibration.cpp index a669c8bf..6c202090 100644 --- a/surround360_render/source/calibration/GeometricCalibration.cpp +++ b/surround360_render/source/calibration/GeometricCalibration.cpp @@ -9,6 +9,7 @@ #include "opencv2/stitching/detail/matchers.hpp" #include "ceres/ceres.h" #include "ceres/rotation.h" +#include "ceres/types.h" #include "GeometricCalibration.h" #include "Camera.h" @@ -40,6 +41,7 @@ DEFINE_double(perturb_principals, 0, "pertub principals (pixels)"); DEFINE_int64(experiments, 1, "calibrate multiple times"); DEFINE_bool(discard_outside_fov, true, "discard matches outside fov"); DEFINE_bool(save_debug_images, false, "save intermediate images"); +//#define FLAGS_save_debug_images false // is the stem expected to be the camera id? i.e. the path is of the form: // / ... /. @@ -767,6 +769,10 @@ void solve( std::vector& rotations, std::vector& traces) { ceres::Solver::Options options; + options.linear_solver_type = ceres::ITERATIVE_SCHUR; + options.preconditioner_type = ceres::SCHUR_JACOBI; + options.num_threads = 32; + options.num_linear_solver_threads = 32; options.use_inner_iterations = true; options.max_num_iterations = 500; options.minimizer_progress_to_stdout = false; @@ -804,13 +810,13 @@ void refine( triangulateTraces(traces, keypointMap, cameras); // visualization for debugging - showMatches( - cameras, - keypointMap, - overlaps, - traces, - debugDir, - pass); + // showMatches( + // cameras, + // keypointMap, + // overlaps, + // traces, + // debugDir, + // pass); // read camera parameters from cameras std::vector positions; diff --git a/surround360_render/source/camera_isp/Raw2Rgb.cpp b/surround360_render/source/camera_isp/Raw2Rgb.cpp index 594447b2..551591f3 100644 --- a/surround360_render/source/camera_isp/Raw2Rgb.cpp +++ b/surround360_render/source/camera_isp/Raw2Rgb.cpp @@ -403,6 +403,8 @@ int main(int argc, char* argv[]) { FLAGS_input_image_path, CV_LOAD_IMAGE_GRAYSCALE | CV_LOAD_IMAGE_ANYDEPTH); + LOG(INFO) << "Camera path " << FLAGS_input_image_path; + if (inputImage.cols > 2 && inputImage.rows > 2) { const uint8_t depth = inputImage.type() & CV_MAT_DEPTH_MASK; diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp index 3a8e362b..71656750 100644 --- a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp @@ -25,7 +25,7 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { rig_.reset(new RigDescription(args_.camera_rig_path())); - num_chunks_ = config.input_columns.size() / 2; + num_chunks_ = config.input_columns.size(); } void new_frame_info() override { @@ -57,47 +57,36 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { void execute(const BatchedColumns& input_columns, BatchedColumns& output_columns) override { - check_frame_info(device_, input_columns[1]); + check_frame(device_, input_columns[0][0]); - i32 input_count = (i32)input_columns[0].rows.size(); + i32 input_count = (i32)num_rows(input_columns[0]); size_t output_image_width = frame_info_.width() * num_chunks_; size_t output_image_height = frame_info_.height(); + output_image_width += (output_image_width % 2); + output_image_height += (output_image_height % 2); size_t output_image_size = output_image_width * output_image_height * 3; - u8 *output_block_buffer = new_block_buffer( - device_, output_image_size * input_count, input_count); - - // Output frame info - scanner::proto::FrameInfo output_frame_info; - output_frame_info.set_width(output_image_width); - output_frame_info.set_height(output_image_height); - output_frame_info.set_channels(3); - u8 *output_frame_info_buffer = new_block_buffer( - device_, output_frame_info.ByteSize(), input_count); - output_frame_info.SerializeToArray(output_frame_info_buffer, - output_frame_info.ByteSize()); + FrameInfo info(output_image_height, output_image_width, 3, FrameType::U8); + std::vector output_frames = new_frames(device_, info, input_count); std::vector pano_chunks(num_chunks_, Mat()); cv::Mat pano; for (i32 i = 0; i < input_count; ++i) { for (i32 c = 0; c < num_chunks_; ++c) { - auto &chunk_col = input_columns[c * 2]; - pano_chunks[c] = cv::Mat(frame_info_.height(), frame_info_.width(), - CV_8UC4, chunk_col.rows[i].buffer); - cv::cvtColor(pano_chunks[c], pano_chunks[c], CV_BGRA2BGR); + auto &chunk_col = input_columns[c]; + cv::cvtColor(frame_to_mat(chunk_col[i].as_const_frame()), + pano_chunks[c], CV_BGRA2BGR); } pano = stackHorizontal(pano_chunks); pano = offsetHorizontalWrap(pano, zeroParallaxNovelViewShiftPixels_); - u8 *output = output_block_buffer + output_image_size * i; + u8 *output = output_frames[i]->data; for (i32 r = 0; r < pano.rows; ++r) { memcpy(output + r * output_image_width * sizeof(char) * 3, pano.data + pano.step * r, pano.cols * sizeof(char) * 3); } - output_columns[0].rows.push_back(Row{output, output_image_size}); - output_columns[1].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + insert_frame(output_columns[0], output_frames[i]); } } @@ -112,7 +101,7 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { REGISTER_OP(ConcatPanoramaChunks) .variadic_inputs() - .outputs({"panorama", "frame_info"}); + .frame_output("panorama"); REGISTER_KERNEL(ConcatPanoramaChunks, ConcatPanoramaChunksKernelCPU) .device(DeviceType::CPU) diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp index 24b2d197..d1ef711e 100644 --- a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp @@ -35,15 +35,14 @@ class ProjectSphericalKernelCPU : public VideoKernel { void execute(const BatchedColumns& input_columns, BatchedColumns& output_columns) override { auto& frame_col = input_columns[0]; - auto& frame_info_col = input_columns[1]; - auto& camera_id_col = input_columns[2]; - check_frame_info(device_, frame_info_col); + auto& camera_id_col = input_columns[1]; + check_frame(device_, frame_col[0]); if (is_reset_) { // Use the new camera id to update the spherical projection parameters is_reset_ = false; - camIdx_ = *((int*)camera_id_col.rows[0].buffer); + camIdx_ = *((int*)camera_id_col[0].buffer); const Camera& camera = rig_->rigSideOnly[camIdx_]; // the negative sign here is so the camera array goes clockwise @@ -55,40 +54,25 @@ class ProjectSphericalKernelCPU : public VideoKernel { bottomAngle_ = -vRadians_ / 2; } - i32 input_count = (i32)frame_col.rows.size(); + i32 input_count = (i32)num_rows(frame_col); size_t output_image_width = args_.eqr_width() * hRadians_ / (2 * M_PI); size_t output_image_height = args_.eqr_height() * vRadians_ / M_PI; size_t output_image_size = output_image_width * output_image_height * 4; - u8 *output_block = - new_block_buffer(device_, output_image_size * input_count, input_count); - - // Output frame info - scanner::proto::FrameInfo output_frame_info; - output_frame_info.set_width(output_image_width); - output_frame_info.set_height(output_image_height); - output_frame_info.set_channels(4); - u8 *output_frame_info_buffer = - new_block_buffer(device_, output_frame_info.ByteSize(), input_count); - output_frame_info.SerializeToArray(output_frame_info_buffer, - output_frame_info.ByteSize()); + FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8); + std::vector output_frames = new_frames(device_, info, input_count); for (i32 i = 0; i < input_count; ++i) { - cv::Mat input(frame_info_.height(), frame_info_.width(), CV_8UC3, - input_columns[0].rows[i].buffer); + cv::Mat input = frame_to_mat(frame_col[i].as_const_frame()); cv::Mat tmp; cv::cvtColor(input, tmp, CV_BGR2BGRA); - cv::Mat projection_image(output_image_height, output_image_width, CV_8UC4, - output_block + i * output_image_size); + cv::Mat projection_image = frame_to_mat(output_frames[i]); surround360::warper::bicubicRemapToSpherical( projection_image, tmp, rig_->rigSideOnly[camIdx_], leftAngle_, rightAngle_, topAngle_, bottomAngle_); - output_columns[0].rows.push_back( - Row{projection_image.data, output_image_size}); - output_columns[1].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + insert_frame(output_columns[0], output_frames[i]); } } @@ -110,8 +94,9 @@ class ProjectSphericalKernelCPU : public VideoKernel { }; REGISTER_OP(ProjectSpherical) - .inputs({"frame", "frame_info", "camera_id"}) - .outputs({"projected_frame", "frame_info"}); + .frame_input("frame") + .input("camera_id") + .frame_output("projected_frame"); REGISTER_KERNEL(ProjectSpherical, ProjectSphericalKernelCPU) .device(DeviceType::CPU) diff --git a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp index ca114e98..9b6c68b8 100644 --- a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp @@ -76,40 +76,23 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { void execute(const BatchedColumns& input_columns, BatchedColumns& output_columns) override { auto& left_frame_col = input_columns[0]; - auto& left_frame_info_col = input_columns[1]; - auto& left_flow_col = input_columns[2]; - auto& left_flow_info_col = input_columns[3]; - - auto& right_frame_col = input_columns[4]; - auto& right_frame_info_col = input_columns[5]; - auto& right_flow_col = input_columns[6]; - auto& right_flow_info_col = input_columns[7]; - check_frame_info(device_, left_frame_info_col); + auto& left_flow_col = input_columns[1]; + auto& right_frame_col = input_columns[2]; + auto& right_flow_col = input_columns[3]; + check_frame(device_, left_frame_col[0]); assert(overlap_image_width_ != -1); - i32 input_count = (i32)left_frame_col.rows.size(); + i32 input_count = (i32)num_rows(left_frame_col); size_t output_image_width = num_novel_views_; size_t output_image_height = frame_info_.height(); size_t output_image_size = output_image_width * output_image_height * 4; - u8 *output_block_buffer = new_block_buffer( - device_, output_image_size * input_count * 2, input_count * 2); - - // Output frame info - scanner::proto::FrameInfo output_frame_info; - output_frame_info.set_width(output_image_width); - output_frame_info.set_height(output_image_height); - output_frame_info.set_channels(3); - u8 *output_frame_info_buffer = new_block_buffer( - device_, output_frame_info.ByteSize(), input_count * 2); - output_frame_info.SerializeToArray(output_frame_info_buffer, - output_frame_info.ByteSize()); + FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8); + std::vector frames = new_frames(device_, info, input_count * 2); for (i32 i = 0; i < input_count; ++i) { - cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - left_frame_col.rows[i].buffer); - cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - right_frame_col.rows[i].buffer); + cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame()); + cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame()); cv::Mat left_overlap_input = left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, overlap_image_width_, left_input.rows)); @@ -118,10 +101,8 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { overlap_image_width_, right_input.rows)); - cv::Mat left_flow(frame_info_.height(), overlap_image_width_, CV_32FC2, - left_flow_col.rows[i].buffer); - cv::Mat right_flow(frame_info_.height(), overlap_image_width_, CV_32FC2, - right_flow_col.rows[i].buffer); + cv::Mat left_flow = frame_to_mat(left_flow_col[i].as_const_frame()); + cv::Mat right_flow = frame_to_mat(right_flow_col[i].as_const_frame()); // Initialize NovelViewGenerator with images and flow since we are // bypassing the prepare method @@ -134,8 +115,8 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { const cv::Mat& chunkL = lazyNovelChunksLR.first; const cv::Mat& chunkR = lazyNovelChunksLR.second; - u8* left_output = output_block_buffer + output_image_size * (2 * i); - u8* right_output = output_block_buffer + output_image_size * (2 * i + 1); + u8* left_output = frames[2 * i]->data; + u8* right_output = frames[2 * i + 1]->data; for (i32 r = 0; r < chunkL.rows; ++r) { memcpy(left_output + r * output_image_width * sizeof(char) * 4, chunkL.data + chunkL.step * r, @@ -145,14 +126,8 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { chunkR.cols * sizeof(char) * 4); } - output_columns[0].rows.push_back( - Row{left_output, output_image_size}); - output_columns[1].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); - output_columns[2].rows.push_back( - Row{right_output, output_image_size}); - output_columns[3].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + insert_frame(output_columns[0], frames[2 * i]); + insert_frame(output_columns[1], frames[2 * i + 1]); } } @@ -170,11 +145,12 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { }; REGISTER_OP(RenderStereoPanoramaChunk) - .inputs({"projected_frame_left", "frame_info_left", "flow_left", - "flow_info_left", "projected_framed_right", "frame_info_right", - "flow_right", "flow_info_right"}) - .outputs({"chunk_left", "frame_info_left", "chunk_right", - "frame_info_right"}); + .frame_input("left_projected_frame") + .frame_input("left_flow") + .frame_input("right_projected_frame") + .frame_input("right_flow") + .frame_output("left_chunk") + .frame_output("right_chunk"); REGISTER_KERNEL(RenderStereoPanoramaChunk, RenderStereoPanoramaChunkKernelCPU) .device(DeviceType::CPU) diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index c4f6f7ed..496b8687 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -54,35 +54,21 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { void execute(const BatchedColumns& input_columns, BatchedColumns& output_columns) override { auto& left_frame_col = input_columns[0]; - auto& left_frame_info_col = input_columns[1]; - auto& right_frame_col = input_columns[2]; - auto& right_frame_info_col = input_columns[3]; - check_frame_info(device_, left_frame_info_col); + auto& right_frame_col = input_columns[1]; + check_frame(device_, left_frame_col[0]); assert(overlap_image_width_ != -1); - i32 input_count = (i32)left_frame_col.rows.size(); + i32 input_count = (i32)num_rows(left_frame_col); size_t output_image_width = overlap_image_width_; size_t output_image_height = frame_info_.height(); size_t output_image_size = output_image_width * output_image_height * 2 * sizeof(float); - u8 *output_block_buffer = new_block_buffer( - device_, output_image_size * input_count * 2, input_count * 2); - - // Output frame info - scanner::proto::FrameInfo output_frame_info; - output_frame_info.set_width(output_image_width); - output_frame_info.set_height(output_image_height); - output_frame_info.set_channels(2); - u8 *output_frame_info_buffer = new_block_buffer( - device_, output_frame_info.ByteSize(), input_count * 2); - output_frame_info.SerializeToArray(output_frame_info_buffer, - output_frame_info.ByteSize()); + FrameInfo info(output_image_height, output_image_width, 2, FrameType::F32); + std::vector frames = new_frames(device_, info, input_count * 2); for (i32 i = 0; i < input_count; ++i) { - cv::Mat left_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - left_frame_col.rows[i].buffer); - cv::Mat right_input(frame_info_.height(), frame_info_.width(), CV_8UC4, - right_frame_col.rows[i].buffer); + cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame()); + cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame()); cv::Mat left_overlap_input = left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, @@ -95,13 +81,13 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { prev_frame_flow_l_to_r_, prev_frame_flow_r_to_l_, prev_overlap_image_l_, prev_overlap_image_r_); - prev_overlap_image_l_ = left_overlap_input; - prev_overlap_image_r_ = right_overlap_input; + left_overlap_input.copyTo(prev_overlap_image_l_); + right_overlap_input.copyTo(prev_overlap_image_r_); prev_frame_flow_l_to_r_ = novel_view_gen_->getFlowLtoR(); prev_frame_flow_r_to_l_ = novel_view_gen_->getFlowRtoL(); - u8* left_output = output_block_buffer + output_image_size * (2 * i); - u8* right_output = output_block_buffer + output_image_size * (2 * i + 1); + u8* left_output = frames[2 * i]->data; + u8* right_output = frames[2 * i + 1]->data; const auto& left_flow = novel_view_gen_->getFlowLtoR(); const auto& right_flow = novel_view_gen_->getFlowRtoL(); for (i32 r = 0; r < left_flow.rows; ++r) { @@ -113,14 +99,8 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { right_flow.cols * sizeof(float) * 2); } - output_columns[0].rows.push_back( - Row{left_output, output_image_size}); - output_columns[1].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); - output_columns[2].rows.push_back( - Row{right_output, output_image_size}); - output_columns[3].rows.push_back( - Row{output_frame_info_buffer, output_frame_info.ByteSize()}); + insert_frame(output_columns[0], frames[2 * i]); + insert_frame(output_columns[1], frames[2 * i + 1]); } } @@ -139,10 +119,10 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { }; REGISTER_OP(TemporalOpticalFlow) - .inputs({"projected_frame_left", "frame_info_left", "projeted_frame_right", - "frame_info_right"}) - .outputs({"flow_left", "frame_info_left", "flow_right", - "frame_info_right"}); + .frame_input("left_projected_frame") + .frame_input("right_projected_frame") + .frame_output("left_flow") + .frame_output("right_flow"); REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) .device(DeviceType::CPU) From 7117b8d967d51cb0a725ade922ceecb9f388a8be Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Thu, 27 Apr 2017 20:00:51 -0400 Subject: [PATCH 13/19] Get rid of pipeline instance restrictions and default to fused 3 --- surround360_render/scripts/scanner_process_video.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index e5c0cf79..291bbf66 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -237,7 +237,7 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): job = Job(columns = [panorama_left, panorama_right], name = 'surround360_pano_both') - return db.run(job, force=True, pipeline_instances_per_node=1) + return db.run(job, force=True) def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, start, end): @@ -310,8 +310,7 @@ def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, job = Job(columns = [panorama_left, panorama_right], name = 'surround360_pano_both') - return db.run(job, pipeline_instances_per_node=1, - force=True) + return db.run(job, force=True) def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): @@ -361,7 +360,7 @@ def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, s name = 'surround360_chunk_{:d}'.format(i)) jobs.append(job) - return db.run(jobs, force=True,pipeline_instances_per_node=1) + return db.run(jobs, force=True) def fused_flow_and_stereo_chunk(db, overlap, render_params): @@ -550,8 +549,8 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): sys.stdout.flush() visualize = False - fused_4 = True # Fused project, flow, stereo chunk, and pano - fused_3 = False # Fused project, flow, and stereo chunk + fused_4 = False # Fused project, flow, stereo chunk, and pano + fused_3 = True # Fused project, flow, and stereo chunk fused_2 = False # Fused flow and stereo chunk fused_1 = False # Nothing fused if fused_4: From 9466c9e70c3cfc16839a7d00c925c3c026816e58 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 21 May 2017 14:56:45 -0400 Subject: [PATCH 14/19] Produce equivalent output in concat kernel, add warmup --- surround360_render/CMakeLists.txt | 3 + .../scripts/scanner_process_video.py | 43 +++++++------ .../concat_panorama_chunks_kernel_cpu.cpp | 64 +++++++++++++++---- .../source/scanner_kernels/surround360.proto | 2 + 4 files changed, 80 insertions(+), 32 deletions(-) diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index d48ab7b0..0b2e0d52 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -6,7 +6,9 @@ OPTION(USE_SCANNER "Enable using Scanner for distributed rendering" ON) FIND_PACKAGE(OpenCV) FIND_PACKAGE(Ceres REQUIRED) +FIND_PACKAGE(Boost REQUIRED) +INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/util) @@ -513,6 +515,7 @@ TARGET_LINK_LIBRARIES( LibJSON gflags glog + -L${Boost_LIBRARY_DIRS} boost_filesystem boost_system double-conversion diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index 291bbf66..f4080b2e 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -208,11 +208,12 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): num_cams = 14 + item_size = 10 left_inputs = [] right_inputs = [] print(chunks) for c in range(num_cams): - left_chunk, right_chunk = chunks[c].as_op().all(item_size=50) + left_chunk, right_chunk = chunks[c].as_op().all(item_size=item_size) left_inputs.append(left_chunk) right_inputs.append(right_chunk) @@ -220,22 +221,17 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): args = db.protobufs.ConcatPanoramaChunksArgs() args.eqr_width = render_params["EQR_WIDTH"] args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - args.zero_parallax_dist = 10000 - args.interpupilary_dist = 6.4 - args.left = is_left - panorama_left = db.ops.ConcatPanoramaChunks(*left_inputs, args=args) - - args = db.protobufs.ConcatPanoramaChunksArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] + args.final_eqr_width = render_params["FINAL_EQR_WIDTH"] + args.final_eqr_height = render_params["FINAL_EQR_HEIGHT"] args.camera_rig_path = render_params["RIG_JSON_FILE"] args.zero_parallax_dist = 10000 args.interpupilary_dist = 6.4 args.left = False - panorama_right = db.ops.ConcatPanoramaChunks(*right_inputs, args=args) - job = Job(columns = [panorama_left, panorama_right], - name = 'surround360_pano_both') + panorama= db.ops.ConcatPanoramaChunks(*(left_inputs + right_inputs), + args=args) + + job = Job(columns = [panorama], + name = 'surround360_pano') return db.run(job, force=True) @@ -315,15 +311,20 @@ def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): jobs = [] - item_size = 11 + warmup_size = 1 + item_size = 15 for i in range(len(videos.tables())): left_cam_idx = i right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) - left_frame = videos.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) - left_cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) - right_frame = videos.tables(right_cam_idx).as_op().range(start, end, item_size = item_size) - right_cam_idx = videos_idx.tables(right_cam_idx).as_op().range(start, end, item_size = item_size) + left_frame = videos.tables(left_cam_idx).as_op().range( + start, end, item_size=item_size, warmup_size=warmup_size) + left_cam_idx = videos_idx.tables(left_cam_idx).as_op().range( + start, end, item_size=item_size, warmup_size=warmup_size) + right_frame = videos.tables(right_cam_idx).as_op().range( + start, end, item_size=item_size, warmup_size=warmup_size) + right_cam_idx = videos_idx.tables(right_cam_idx).as_op().range( + start, end, item_size=item_size, warmup_size=warmup_size) args = db.protobufs.ProjectSphericalArgs() args.eqr_width = render_params["EQR_WIDTH"] @@ -572,6 +573,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): True) print('Concat', timer() - concat_start) chunk_col[0].profiler().write_trace('fused3.trace') + pano_col.profiler().write_trace('fused3_concat.trace') elif fused_2: proj_start = timer() proj_col = project_images(db, videos, videos_idx, render_params) @@ -601,11 +603,12 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) png_start = timer() - pano_col = db.table('surround360_pano_both') + pano_col = db.table('surround360_pano') # left_table = pano_col[0] # right_table = pano_col[1] if visualize: - left_table.columns('panorama').save_mp4('pano_test.mp4') + pano_col.columns('panorama').save_mp4('pano_test.mp4', + scale=(2048, 2048)) print('To png', timer() - png_start) end_time = timer() diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp index 71656750..bde11a67 100644 --- a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp @@ -11,6 +11,21 @@ using namespace scanner; namespace surround360 { +namespace { +void padToheight(Mat& unpaddedImage, const int targetHeight) { + const int paddingAbove = (targetHeight - unpaddedImage.rows) / 2; + const int paddingBelow = targetHeight - unpaddedImage.rows - paddingAbove; + cv::copyMakeBorder( + unpaddedImage, + unpaddedImage, + paddingAbove, + paddingBelow, + 0, + 0, + BORDER_CONSTANT, + Scalar(0.0, 0.0, 0.0)); +} +} using namespace optical_flow; using namespace math_util; @@ -25,7 +40,7 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { rig_.reset(new RigDescription(args_.camera_rig_path())); - num_chunks_ = config.input_columns.size(); + num_chunks_ = config.input_columns.size() / 2; } void new_frame_info() override { @@ -60,8 +75,8 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { check_frame(device_, input_columns[0][0]); i32 input_count = (i32)num_rows(input_columns[0]); - size_t output_image_width = frame_info_.width() * num_chunks_; - size_t output_image_height = frame_info_.height(); + size_t output_image_width = (size_t)args_.final_eqr_width(); + size_t output_image_height = (size_t)args_.final_eqr_height(); output_image_width += (output_image_width % 2); output_image_height += (output_image_height % 2); size_t output_image_size = @@ -69,21 +84,46 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { FrameInfo info(output_image_height, output_image_width, 3, FrameType::U8); std::vector output_frames = new_frames(device_, info, input_count); - std::vector pano_chunks(num_chunks_, Mat()); - cv::Mat pano; + std::vector left_pano_chunks(num_chunks_, Mat()); + std::vector right_pano_chunks(num_chunks_, Mat()); + cv::Mat left_pano; + cv::Mat right_pano; for (i32 i = 0; i < input_count; ++i) { for (i32 c = 0; c < num_chunks_; ++c) { - auto &chunk_col = input_columns[c]; - cv::cvtColor(frame_to_mat(chunk_col[i].as_const_frame()), - pano_chunks[c], CV_BGRA2BGR); + auto &left_chunk_col = input_columns[c]; + cv::cvtColor(frame_to_mat(left_chunk_col[i].as_const_frame()), + left_pano_chunks[c], CV_BGRA2BGR); + + auto &right_chunk_col = input_columns[num_chunks_ + c]; + cv::cvtColor(frame_to_mat(right_chunk_col[i].as_const_frame()), + right_pano_chunks[c], CV_BGRA2BGR); } - pano = stackHorizontal(pano_chunks); - pano = offsetHorizontalWrap(pano, zeroParallaxNovelViewShiftPixels_); + left_pano = stackHorizontal(left_pano_chunks); + left_pano = + offsetHorizontalWrap(left_pano, zeroParallaxNovelViewShiftPixels_); + + right_pano = stackHorizontal(right_pano_chunks); + right_pano = + offsetHorizontalWrap(right_pano, -zeroParallaxNovelViewShiftPixels_); + + padToheight(left_pano, args_.eqr_height()); + padToheight(right_pano, args_.eqr_height()); + + resize(left_pano, left_pano, + Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0, + INTER_CUBIC); + resize(right_pano, right_pano, + Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0, + INTER_CUBIC); + + cv::Mat stereo_equirect = + stackVertical(vector({left_pano, right_pano})); u8 *output = output_frames[i]->data; - for (i32 r = 0; r < pano.rows; ++r) { + for (i32 r = 0; r < stereo_equirect.rows; ++r) { memcpy(output + r * output_image_width * sizeof(char) * 3, - pano.data + pano.step * r, pano.cols * sizeof(char) * 3); + stereo_equirect.data + stereo_equirect.step * r, + stereo_equirect.cols * sizeof(char) * 3); } insert_frame(output_columns[0], output_frames[i]); diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto index 04652521..1afc20fb 100644 --- a/surround360_render/source/scanner_kernels/surround360.proto +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -25,6 +25,8 @@ message RenderStereoPanoramaChunkArgs { message ConcatPanoramaChunksArgs { int32 eqr_height = 1; int32 eqr_width = 2; + int32 final_eqr_height = 7; + int32 final_eqr_width = 8; string camera_rig_path = 3; float zero_parallax_dist = 4; float interpupilary_dist = 5; From 2875ac1fb7c6ca953056798756a0edf62a1f2572 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Sun, 21 May 2017 16:35:53 -0400 Subject: [PATCH 15/19] Add specifying # cores for scanner benchmark --- surround360_render/scripts/batch_process_video.py | 1 + .../scripts/scanner_process_video.py | 15 +++++++++++---- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/surround360_render/scripts/batch_process_video.py b/surround360_render/scripts/batch_process_video.py index 7a171b66..bfa927b4 100755 --- a/surround360_render/scripts/batch_process_video.py +++ b/surround360_render/scripts/batch_process_video.py @@ -82,6 +82,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l parser.add_argument('--rig_json_file', help='path to rig json file', required=True) parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True) parser.add_argument('--verbose', dest='verbose', action='store_true') + parser.add_argument('--cores', default=-1) parser.set_defaults(save_debug_images=False) parser.set_defaults(enable_top=False) parser.set_defaults(enable_bottom=False) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index f4080b2e..f21b8526 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -233,7 +233,8 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): job = Job(columns = [panorama], name = 'surround360_pano') - return db.run(job, force=True) + return db.run(job, force=True, + pipeline_instances_per_node=render_params["CORES"]) def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, start, end): @@ -361,7 +362,8 @@ def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, s name = 'surround360_chunk_{:d}'.format(i)) jobs.append(job) - return db.run(jobs, force=True) + return db.run(jobs, force=True, + pipeline_instances_per_node=render_params["CORES"]) def fused_flow_and_stereo_chunk(db, overlap, render_params): @@ -415,6 +417,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): parser.add_argument('--rig_json_file', help='path to rig json file', required=True) parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True) parser.add_argument('--verbose', dest='verbose', action='store_true') + parser.add_argument('--cores', default=-1, type=int) parser.set_defaults(save_debug_images=False) parser.set_defaults(enable_top=False) parser.set_defaults(enable_bottom=False) @@ -462,6 +465,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): "POLAR_FLOW_ALGORITHM": flow_alg, "POLEREMOVAL_FLOW_ALGORITHM": flow_alg, "EXTRA_FLAGS": "", + "CORES": args["cores"] } @@ -514,8 +518,11 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): db_start = timer() with Database() as db: print('DB', timer() - db_start) - db.load_op('build/lib/libsurround360kernels.so', - 'build/source/scanner_kernels/surround360_pb2.py') + db.load_op( + os.path.join(surround360_render_dir, + 'build/lib/libsurround360kernels.so'), + os.path.join(surround360_render_dir, + 'build/source/scanner_kernels/surround360_pb2.py')) if not db.has_collection(collection_name): print "----------- [Render] loading surround360 collection" From aed970e6a32a6cac38799f41fcd5395ab756a34a Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Mon, 4 Sep 2017 11:26:33 -0400 Subject: [PATCH 16/19] Update to new scanner kernel interface --- .../concat_panorama_chunks_kernel_cpu.cpp | 10 ++++------ .../scanner_kernels/project_spherical_kernel_cpu.cpp | 10 ++++------ .../render_stereo_panorama_chunk_kernel_cpu.cpp | 12 +++++------- .../temporal_optical_flow_kernel_cpu.cpp | 10 ++++------ 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp index bde11a67..b0d24c08 100644 --- a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp @@ -30,12 +30,11 @@ void padToheight(Mat& unpaddedImage, const int targetHeight) { using namespace optical_flow; using namespace math_util; -class ConcatPanoramaChunksKernelCPU : public VideoKernel { +class ConcatPanoramaChunksKernelCPU : public BatchedKernel, public VideoKernel { public: - ConcatPanoramaChunksKernelCPU(const Kernel::Config& config) - : VideoKernel(config), - device_(config.devices[0]), - work_item_size_(config.work_item_size) { + ConcatPanoramaChunksKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { args_.ParseFromArray(config.args.data(), config.args.size()); rig_.reset(new RigDescription(args_.camera_rig_path())); @@ -134,7 +133,6 @@ class ConcatPanoramaChunksKernelCPU : public VideoKernel { surround360::proto::ConcatPanoramaChunksArgs args_; std::unique_ptr rig_; DeviceHandle device_; - int work_item_size_; int num_chunks_; float zeroParallaxNovelViewShiftPixels_; }; diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp index d1ef711e..e7ea48ea 100644 --- a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp @@ -13,12 +13,11 @@ using namespace scanner; namespace surround360 { -class ProjectSphericalKernelCPU : public VideoKernel { +class ProjectSphericalKernelCPU : public BatchedKernel, public VideoKernel { public: - ProjectSphericalKernelCPU(const Kernel::Config& config) - : VideoKernel(config), - device_(config.devices[0]), - work_item_size_(config.work_item_size) { + ProjectSphericalKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { args_.ParseFromArray(config.args.data(), config.args.size()); // Initialize camera rig @@ -80,7 +79,6 @@ class ProjectSphericalKernelCPU : public VideoKernel { surround360::proto::ProjectSphericalArgs args_; std::unique_ptr rig_; DeviceHandle device_; - i32 work_item_size_; bool is_reset_ = true; float hRadians_; diff --git a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp index 9b6c68b8..c95b81ca 100644 --- a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp @@ -15,12 +15,11 @@ namespace surround360 { using namespace optical_flow; using namespace math_util; -class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { - public: - RenderStereoPanoramaChunkKernelCPU(const Kernel::Config& config) - : VideoKernel(config), - device_(config.devices[0]), - work_item_size_(config.work_item_size) { +class RenderStereoPanoramaChunkKernelCPU : public BatchedKernel, + public VideoKernel { +public: + RenderStereoPanoramaChunkKernelCPU(const KernelConfig &config) + : BatchedKernel(config), device_(config.devices[0]) { args_.ParseFromArray(config.args.data(), config.args.size()); rig_.reset(new RigDescription(args_.camera_rig_path())); @@ -135,7 +134,6 @@ class RenderStereoPanoramaChunkKernelCPU : public VideoKernel { surround360::proto::RenderStereoPanoramaChunkArgs args_; std::unique_ptr rig_; DeviceHandle device_; - int work_item_size_; float fov_horizontal_radians_; int overlap_image_width_; int num_novel_views_; diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index 496b8687..3f9fc7b4 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -15,12 +15,11 @@ namespace surround360 { using namespace optical_flow; using namespace math_util; -class TemporalOpticalFlowKernelCPU : public VideoKernel { +class TemporalOpticalFlowKernelCPU : public BatchedKernel, public VideoKernel { public: - TemporalOpticalFlowKernelCPU(const Kernel::Config& config) - : VideoKernel(config), - device_(config.devices[0]), - work_item_size_(config.work_item_size) { + TemporalOpticalFlowKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { args_.ParseFromArray(config.args.data(), config.args.size()); rig_.reset(new RigDescription(args_.camera_rig_path())); @@ -108,7 +107,6 @@ class TemporalOpticalFlowKernelCPU : public VideoKernel { surround360::proto::TemporalOpticalFlowArgs args_; std::unique_ptr rig_; DeviceHandle device_; - i32 work_item_size_; int overlap_image_width_; std::unique_ptr novel_view_gen_; From 9f016f1dbcab3d4388de35c24c908efe5c264fd1 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Mon, 4 Sep 2017 16:46:14 -0400 Subject: [PATCH 17/19] Update item_size to task_size --- .../scripts/scanner_process_video.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index f21b8526..c9160389 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -213,7 +213,7 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): right_inputs = [] print(chunks) for c in range(num_cams): - left_chunk, right_chunk = chunks[c].as_op().all(item_size=item_size) + left_chunk, right_chunk = chunks[c].as_op().all(task_size=item_size) left_inputs.append(left_chunk) right_inputs.append(right_chunk) @@ -244,8 +244,8 @@ def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, proj_frames = [] for i in range(len(videos.tables())): left_cam_idx = i - frame = videos.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) - cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, item_size = item_size) + frame = videos.tables(left_cam_idx).as_op().range(start, end, task_size=item_size) + cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, task_size=item_size) args = db.protobufs.ProjectSphericalArgs() args.eqr_width = render_params["EQR_WIDTH"] @@ -313,19 +313,19 @@ def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): jobs = [] warmup_size = 1 - item_size = 15 + item_size = 1 for i in range(len(videos.tables())): left_cam_idx = i right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) left_frame = videos.tables(left_cam_idx).as_op().range( - start, end, item_size=item_size, warmup_size=warmup_size) + start, end, task_size=item_size, warmup_size=warmup_size) left_cam_idx = videos_idx.tables(left_cam_idx).as_op().range( - start, end, item_size=item_size, warmup_size=warmup_size) + start, end, task_size=item_size, warmup_size=warmup_size) right_frame = videos.tables(right_cam_idx).as_op().range( - start, end, item_size=item_size, warmup_size=warmup_size) + start, end, task_size=item_size, warmup_size=warmup_size) right_cam_idx = videos_idx.tables(right_cam_idx).as_op().range( - start, end, item_size=item_size, warmup_size=warmup_size) + start, end, task_size=item_size, warmup_size=warmup_size) args = db.protobufs.ProjectSphericalArgs() args.eqr_width = render_params["EQR_WIDTH"] From c7c9debcaaada59bc92c9e94a101294ba93140a0 Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Tue, 17 Oct 2017 14:38:52 -0400 Subject: [PATCH 18/19] Update fused_3 to new scanner api --- .../scripts/scanner_process_video.py | 147 +++++++++++------- .../temporal_optical_flow_kernel_cpu.cpp | 3 +- 2 files changed, 91 insertions(+), 59 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index c9160389..beec0179 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -1,4 +1,4 @@ -from scannerpy import Database, DeviceType, Job +from scannerpy import Database, DeviceType, Job, BulkJob from scannerpy.stdlib import NetDescriptor, parsers, bboxes import numpy as np import argparse @@ -209,15 +209,17 @@ def render_stereo_panorama_chunks(db, overlap, flow, render_params): def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): num_cams = 14 item_size = 10 + print(chunks) + assert num_cams == len(chunks) + left_inputs = [] right_inputs = [] - print(chunks) for c in range(num_cams): - left_chunk, right_chunk = chunks[c].as_op().all(task_size=item_size) + left_chunk = db.ops.FrameInput() + right_chunk = db.ops.FrameInput() left_inputs.append(left_chunk) right_inputs.append(right_chunk) - jobs = [] args = db.protobufs.ConcatPanoramaChunksArgs() args.eqr_width = render_params["EQR_WIDTH"] args.eqr_height = render_params["EQR_HEIGHT"] @@ -227,14 +229,27 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): args.zero_parallax_dist = 10000 args.interpupilary_dist = 6.4 args.left = False - panorama= db.ops.ConcatPanoramaChunks(*(left_inputs + right_inputs), - args=args) + panorama = db.ops.ConcatPanoramaChunks( + *(left_inputs + right_inputs), + args=args) + output_op = db.ops.Output(columns=[panorama]) + + op_args = { + output_op: 'surround360_pano' + } + for c in range(num_cams): + op_args[left_chunk[c]] = chunks[c].column('left_chunk') + op_args[right_chunk[c]] = chunks[c].column('right_chunk') - job = Job(columns = [panorama], - name = 'surround360_pano') + job = Job(op_args=op_args) + bulk_job = BulkJob(output=output_op, jobs=[job]) - return db.run(job, force=True, - pipeline_instances_per_node=render_params["CORES"]) + tables = db.run( + bulk_job, force=True, + work_packet_size=item_size, + io_packet_size=item_size, + pipeline_instances_per_node=render_params["CORES"]) + return tables[0] def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, start, end): @@ -311,58 +326,75 @@ def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): - jobs = [] - warmup_size = 1 - item_size = 1 - for i in range(len(videos.tables())): - left_cam_idx = i - right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + warmup_size = 10 + task_size = 100 - left_frame = videos.tables(left_cam_idx).as_op().range( - start, end, task_size=item_size, warmup_size=warmup_size) - left_cam_idx = videos_idx.tables(left_cam_idx).as_op().range( - start, end, task_size=item_size, warmup_size=warmup_size) - right_frame = videos.tables(right_cam_idx).as_op().range( - start, end, task_size=item_size, warmup_size=warmup_size) - right_cam_idx = videos_idx.tables(right_cam_idx).as_op().range( - start, end, task_size=item_size, warmup_size=warmup_size) + left_frame = db.ops.FrameInput() + left_cam_idx = db.ops.Input() - args = db.protobufs.ProjectSphericalArgs() - args.eqr_width = render_params["EQR_WIDTH"] - args.eqr_height = render_params["EQR_HEIGHT"] - args.camera_rig_path = render_params["RIG_JSON_FILE"] - left_proj_frame = db.ops.ProjectSpherical( - frame = left_frame, - camera_id = left_cam_idx, - args=args) - right_proj_frame = db.ops.ProjectSpherical( - frame = right_frame, - camera_id = right_cam_idx, - args=args) + right_frame = db.ops.FrameInput() + right_cam_idx = db.ops.Input() - left_flow, right_flow = db.ops.TemporalOpticalFlow( - left_projected_frame = left_proj_frame, - right_projected_frame = right_proj_frame, - flow_algo = render_params["SIDE_FLOW_ALGORITHM"], - camera_rig_path = render_params["RIG_JSON_FILE"]) + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + left_proj_frame = db.ops.ProjectSpherical( + frame = left_frame, + camera_id = left_cam_idx, + args=args) + right_proj_frame = db.ops.ProjectSpherical( + frame = right_frame, + camera_id = right_cam_idx, + args=args) - left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( - left_projected_frame = left_proj_frame, - left_flow = left_flow, - right_projected_frame = right_proj_frame, - right_flow = right_flow, - eqr_width = render_params["EQR_WIDTH"], - eqr_height = render_params["EQR_HEIGHT"], - camera_rig_path = render_params["RIG_JSON_FILE"], - flow_algo = render_params["SIDE_FLOW_ALGORITHM"], - zero_parallax_dist = 10000, - interpupilary_dist = 6.4) + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + camera_rig_path = render_params["RIG_JSON_FILE"], + warmup = warmup_size) + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + + left_chunk_sample = left_chunk.sample() + right_chunk_sample = right_chunk.sample() + + output_op = db.ops.Output(columns=[left_chunk_sample, right_chunk_sample]) - job = Job(columns = [left_chunk.lossless(), right_chunk.lossless()], - name = 'surround360_chunk_{:d}'.format(i)) + jobs = [] + for i in range(len(videos.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + + sample = db.sampler.range(start, end) + + job = Job(op_args={ + left_frame: videos.tables(left_cam_idx).column('frame'), + left_frame_idx: videos_idx.tables(left_cam_idx).column('camera_index'), + right_frame: videos.tables(right_cam_idx).column('frame'), + right_frame_idx: videos_idx.tables(right_cam_idx).column('camera_index'), + left_chunk_sample: sample, + right_chunk_sample: sample, + output_op: 'surround360_chunk_{:d}'.format(i), + }) jobs.append(job) - return db.run(jobs, force=True, + bulk_job = BulkJob(output=output_op, jobs=jobs) + + return db.run(bulk_job, force=True, + work_packet_size=10, + io_packet_size=task_size, pipeline_instances_per_node=render_params["CORES"]) @@ -568,9 +600,8 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): pano_col.profiler().write_trace('fused4.trace') elif fused_3: flow_stereo_start = timer() - chunk_col = fused_project_flow_and_stereo_chunk(db, videos, videos_idx, - render_params, - min_frame, max_frame + 1) + chunk_col = fused_project_flow_and_stereo_chunk( + db, videos, videos_idx, render_params, min_frame, max_frame + 1) print(db.summarize()) chunk_col = [db.table('surround360_chunk_{:d}'.format(i)) for i in range(14)] diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp index 3f9fc7b4..160d3cd3 100644 --- a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -120,7 +120,8 @@ REGISTER_OP(TemporalOpticalFlow) .frame_input("left_projected_frame") .frame_input("right_projected_frame") .frame_output("left_flow") - .frame_output("right_flow"); + .frame_output("right_flow") + .bounded_state(); REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) .device(DeviceType::CPU) From 5b52f8a8e622de736cf4c8664ab2ed20b5979e9a Mon Sep 17 00:00:00 2001 From: Alex Poms Date: Thu, 19 Oct 2017 02:15:55 -0400 Subject: [PATCH 19/19] Fix minor bugs in using scanner interface --- .../scripts/scanner_process_video.py | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py index beec0179..eab35b82 100644 --- a/surround360_render/scripts/scanner_process_video.py +++ b/surround360_render/scripts/scanner_process_video.py @@ -238,8 +238,8 @@ def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): output_op: 'surround360_pano' } for c in range(num_cams): - op_args[left_chunk[c]] = chunks[c].column('left_chunk') - op_args[right_chunk[c]] = chunks[c].column('right_chunk') + op_args[left_inputs[c]] = chunks[c].column('left_chunk') + op_args[right_inputs[c]] = chunks[c].column('right_chunk') job = Job(op_args=op_args) bulk_job = BulkJob(output=output_op, jobs=[job]) @@ -370,20 +370,21 @@ def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, s left_chunk_sample = left_chunk.sample() right_chunk_sample = right_chunk.sample() - output_op = db.ops.Output(columns=[left_chunk_sample, right_chunk_sample]) + output_op = db.ops.Output(columns=[left_chunk_sample.lossless(), + right_chunk_sample.lossless()]) jobs = [] for i in range(len(videos.tables())): - left_cam_idx = i - right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + left_idx = i + right_idx = (left_idx + 1) % len(videos.tables()) sample = db.sampler.range(start, end) job = Job(op_args={ - left_frame: videos.tables(left_cam_idx).column('frame'), - left_frame_idx: videos_idx.tables(left_cam_idx).column('camera_index'), - right_frame: videos.tables(right_cam_idx).column('frame'), - right_frame_idx: videos_idx.tables(right_cam_idx).column('camera_index'), + left_frame: videos.tables(left_idx).column('frame'), + left_cam_idx: videos_idx.tables(left_idx).column('camera_index'), + right_frame: videos.tables(right_idx).column('frame'), + right_cam_idx: videos_idx.tables(right_idx).column('camera_index'), left_chunk_sample: sample, right_chunk_sample: sample, output_op: 'surround360_chunk_{:d}'.format(i), @@ -564,7 +565,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): collection, _ = db.ingest_video_collection(collection_name, paths, force=True) - idx_table_names = [] + idx_tables = [] num_rows = collection.tables(0).num_rows() columns = ['camera_index'] for c in range(0, len(paths)): @@ -573,9 +574,8 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): rows.append([struct.pack('i', c)]) table_name = 'surround_cam_idx_{:d}'.format(c) db.new_table(table_name, columns, rows, force=True) - idx_table_names.append(table_name) - db.new_collection(idx_collection_name, idx_table_names, - force=True) + idx_tables.append(db.table(table_name)) + db.new_collection(idx_collection_name, idx_tables, force=True) videos = db.collection(collection_name) @@ -588,7 +588,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): print(render_params) sys.stdout.flush() - visualize = False + visualize = True fused_4 = False # Fused project, flow, stereo chunk, and pano fused_3 = True # Fused project, flow, and stereo chunk fused_2 = False # Fused flow and stereo chunk @@ -645,7 +645,7 @@ def fused_flow_and_stereo_chunk(db, overlap, render_params): # left_table = pano_col[0] # right_table = pano_col[1] if visualize: - pano_col.columns('panorama').save_mp4('pano_test.mp4', + pano_col.column('panorama').save_mp4('pano_test', scale=(2048, 2048)) print('To png', timer() - png_start)