diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt
index 1ee7c391..0b2e0d52 100644
--- a/surround360_render/CMakeLists.txt
+++ b/surround360_render/CMakeLists.txt
@@ -2,9 +2,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.2)
PROJECT(Surround360Render CXX)
+OPTION(USE_SCANNER "Enable using Scanner for distributed rendering" ON)
+
FIND_PACKAGE(OpenCV)
FIND_PACKAGE(Ceres REQUIRED)
+FIND_PACKAGE(Boost REQUIRED)
+INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS})
INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS})
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source)
INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/util)
@@ -15,6 +19,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/render)
INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR})
INCLUDE_DIRECTORIES(/usr/local/include/eigen3)
+SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC")
+
if (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
# required for clang, including AppleClang
LINK_DIRECTORIES(/usr/local/lib)
@@ -53,7 +59,7 @@ ADD_CUSTOM_TARGET( _source SOURCES ${SRC} )
### Halide support
IF (DEFINED HALIDE_DIR)
- INCLUDE("${HALIDE_DIR}/../HalideGenerator.cmake")
+ INCLUDE("${HALIDE_DIR}/HalideGenerator.cmake")
SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_HALIDE")
@@ -90,14 +96,19 @@ FILE(GLOB optical_flow_SRC "source/optical_flow/*.cpp")
FILE(GLOB render_SRC "source/render/*.cpp")
FILE(GLOB util_SRC "source/util/*.cpp")
-ADD_LIBRARY(
- LibVrCamera
+ADD_LIBRARY(LibVrCamera
source/util/SystemUtil.cpp
${calibration_SRC}
${optical_flow_SRC}
${render_SRC}
${util_SRC}
)
+# TARGET_LINK_LIBRARIES(LibVrCamera
+# folly
+# ceres
+# LibJSON
+# ${OpenCV_LIBS}
+# ${PLATFORM_SPECIFIC_LIBS})
TARGET_COMPILE_FEATURES(LibVrCamera PRIVATE cxx_range_for)
### Raw2Rgb ###
@@ -198,6 +209,12 @@ TARGET_LINK_LIBRARIES(
LibJSON
glog
gflags
+
+# folly
+# ceres
+# LibJSON
+ ${OpenCV_LIBS}
+ ${PLATFORM_SPECIFIC_LIBS}
)
### TestOpticalFlow ###
@@ -498,11 +515,31 @@ TARGET_LINK_LIBRARIES(
LibJSON
gflags
glog
- folly
+ -L${Boost_LIBRARY_DIRS}
boost_filesystem
boost_system
double-conversion
+ folly
+ ${CERES_LIBRARIES}
${OpenCV_LIBS}
${PLATFORM_SPECIFIC_LIBS}
- ${CERES_LIBRARIES}
)
+
+
+IF (USE_SCANNER)
+ SET(SCANNER_PATH "~/repos/scanner/")
+ include(${SCANNER_PATH}/cmake/Util/Op.cmake)
+ build_op(
+ LIB_NAME surround360kernels
+ CPP_SRCS
+ source/scanner_kernels/project_spherical_kernel_cpu.cpp
+ source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp
+ source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp
+ source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp
+
+ PROTO_SRC source/scanner_kernels/surround360.proto)
+ target_link_libraries(surround360kernels PUBLIC
+ LibVrCamera
+ folly
+ ${OpenCV_LIBS})
+ENDIF()
diff --git a/surround360_render/scripts/batch_process_video.py b/surround360_render/scripts/batch_process_video.py
index 6be7741e..bfa927b4 100755
--- a/surround360_render/scripts/batch_process_video.py
+++ b/surround360_render/scripts/batch_process_video.py
@@ -82,6 +82,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l
parser.add_argument('--rig_json_file', help='path to rig json file', required=True)
parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True)
parser.add_argument('--verbose', dest='verbose', action='store_true')
+ parser.add_argument('--cores', default=-1)
parser.set_defaults(save_debug_images=False)
parser.set_defaults(enable_top=False)
parser.set_defaults(enable_bottom=False)
@@ -192,6 +193,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l
render_params["FINAL_EQR_HEIGHT"] = 6144
elif quality == "8k":
render_params["SHARPENNING"] = 0.25
+ render_params["SHARPENNING"] = 0.0
render_params["EQR_WIDTH"] = 8400
render_params["EQR_HEIGHT"] = 4096
render_params["FINAL_EQR_WIDTH"] = 8192
diff --git a/surround360_render/scripts/geometric_calibration.py b/surround360_render/scripts/geometric_calibration.py
index ead2034d..cb907b50 100644
--- a/surround360_render/scripts/geometric_calibration.py
+++ b/surround360_render/scripts/geometric_calibration.py
@@ -36,6 +36,7 @@
{COLMAP_DIR}/feature_extractor
--General.image_path {IMAGE_PATH}
--General.database_path {COLMAP_DB_PATH}
+--ExtractionOptions.gpu_index 0
"""
COLMAP_MATCH_TEMPLATE = """
diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py
new file mode 100644
index 00000000..eab35b82
--- /dev/null
+++ b/surround360_render/scripts/scanner_process_video.py
@@ -0,0 +1,659 @@
+from scannerpy import Database, DeviceType, Job, BulkJob
+from scannerpy.stdlib import NetDescriptor, parsers, bboxes
+import numpy as np
+import argparse
+import os
+import signal
+import subprocess
+import sys
+import struct
+import scipy.misc
+import time
+
+from os import listdir, system
+from os.path import isfile, join
+from timeit import default_timer as timer
+
+current_process = None
+def signal_term_handler(signal, frame):
+ if current_process:
+ print "Terminating process: " + current_process.name + "..."
+ current_process.terminate()
+ sys.exit(0)
+
+RENDER_COMMAND_TEMPLATE = """
+{SURROUND360_RENDER_DIR}/bin/TestRenderStereoPanorama
+--logbuflevel -1
+--log_dir {LOG_DIR}
+--stderrthreshold 0
+--v {VERBOSE_LEVEL}
+--rig_json_file {RIG_JSON_FILE}
+--imgs_dir {SRC_DIR}/rgb
+--frame_number {FRAME_ID}
+--output_data_dir {SRC_DIR}
+--prev_frame_data_dir {PREV_FRAME_DIR}
+--output_cubemap_path {OUT_CUBE_DIR}/cube_{FRAME_ID}.png
+--output_equirect_path {OUT_EQR_DIR}/eqr_{FRAME_ID}.png
+--cubemap_format {CUBEMAP_FORMAT}
+--side_flow_alg {SIDE_FLOW_ALGORITHM}
+--polar_flow_alg {POLAR_FLOW_ALGORITHM}
+--poleremoval_flow_alg {POLEREMOVAL_FLOW_ALGORITHM}
+--cubemap_width {CUBEMAP_WIDTH}
+--cubemap_height {CUBEMAP_HEIGHT}
+--eqr_width {EQR_WIDTH}
+--eqr_height {EQR_HEIGHT}
+--final_eqr_width {FINAL_EQR_WIDTH}
+--final_eqr_height {FINAL_EQR_HEIGHT}
+--interpupilary_dist 6.4
+--zero_parallax_dist 10000
+--sharpenning {SHARPENNING}
+{EXTRA_FLAGS}
+"""
+
+def start_subprocess(name, cmd):
+ global current_process
+ current_process = subprocess.Popen(cmd, shell=True)
+ current_process.name = name
+ current_process.communicate()
+
+
+def project_tasks(db, videos, videos_idx):
+ tasks = []
+ sampler_args = db.protobufs.AllSamplerArgs()
+ sampler_args.warmup_size = 0
+ sampler_args.sample_size = 32
+
+ for i, (vid, vid_idx) in enumerate(zip(videos.tables(),
+ videos_idx.tables())):
+ task = db.protobufs.Task()
+ task.output_table_name = vid.name() + str(i)
+
+ sample = task.samples.add()
+ sample.table_name = vid.name()
+ sample.column_names.extend([c.name() for c in vid.columns()])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+
+ sample = task.samples.add()
+ sample.table_name = vid_idx.name()
+ sample.column_names.extend(['camera_index'])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+ tasks.append(task)
+ return tasks
+
+
+def project_tasks_join(db, videos, videos_idx):
+ tasks = []
+ sampler_args = db.protobufs.AllSamplerArgs()
+ sampler_args.warmup_size = 0
+ sampler_args.sample_size = 32
+
+ for i in range(len(videos.tables())):
+ left_cam_idx = i
+ if i == len(videos.tables()) - 1:
+ right_cam_idx = 0
+ else:
+ right_cam_idx = left_cam_idx + 1
+
+ vid_left = videos.tables(left_cam_idx)
+ vid_idx_left = videos_idx.tables(left_cam_idx)
+ vid_right = videos.tables(right_cam_idx)
+ vid_idx_right = videos_idx.tables(right_cam_idx)
+
+ task = db.protobufs.Task()
+ task.output_table_name = vid_left.name() + str(i)
+
+ sample = task.samples.add()
+ sample.table_name = vid_left.name()
+ sample.column_names.extend([c.name() for c in vid_left.columns()])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+
+ sample = task.samples.add()
+ sample.table_name = vid_idx_left.name()
+ sample.column_names.extend(['camera_index'])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+
+ sample = task.samples.add()
+ sample.table_name = vid_right.name()
+ sample.column_names.extend(['frame'])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+
+ sample = task.samples.add()
+ sample.table_name = vid_idx_right.name()
+ sample.column_names.extend(['camera_index'])
+ sample.sampling_function = "All"
+ sample.sampling_args = sampler_args.SerializeToString()
+
+ tasks.append(task)
+ return tasks
+
+
+def project_images(db, videos, videos_idx, render_params):
+ jobs = []
+ for i in range(len(video.tables())):
+ left_frame = videos.tables(i).as_op().all()
+ left_cam_idx = videos_idx.tables(i).as_op().all()
+
+ args = db.protobufs.ProjectSphericalArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ proj_frame = db.ops.ProjectSpherical(frame = frame, camra_id = cam_idx,
+ args=args)
+
+ job = Job(columns = [proj_frame],
+ name = 'surround360_spherical_{:d}'.format(i))
+ jobs.append(job)
+
+ return db.run(jobs, 'surround360_spherical', force=True)
+
+
+def compute_temporal_flow(db, overlap, render_params):
+ jobs = []
+ for i in range(len(overlap.tables())):
+ left_cam_idx = i
+ right_cam_idx = (left_cam_idx + 1) % len(overlap.tables())
+
+ left_proj_frame = overlap.tables(left_cam_idx).as_op().all()
+ right_proj_frame = overlap.tables(right_cam_idx).as_op().all()
+
+ args = db.protobufs.TemporalOpticalFlowArgs()
+ args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ left_flow, right_flow = db.ops.TemporalOpticalFlow(
+ left_projected_frame = left_proj_frame,
+ right_projected_frame = right_proj_frame,
+ args=args)
+
+ job = Job(columns = [left_flow, right_flow],
+ name = 'surround360_flow_{:d}'.format(i))
+ jobs.append(job)
+
+ return db.run(jobs, 'surround360_flow', force=True)
+
+
+def render_stereo_panorama_chunks(db, overlap, flow, render_params):
+ jobs = []
+ for i in range(len(video.tables())):
+ left_cam_idx = i
+ right_cam_idx = (left_cam_idx + 1) % len(video.tables())
+
+ left_proj_frame = overlap.tables(left_cam_idx).as_op().all()
+ left_flow = flow.tables(left_cam_idx).as_op().all()
+ right_proj_frame = videos.tables(right_cam_idx).as_op().all()
+ right_flow = flow_idx.tables(right_cam_idx).as_op().all()
+
+ left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk(
+ left_projected_frame = left_proj_frame,
+ left_flow = left_flow,
+ right_projected_frame = right_proj_frame,
+ right_flow = right_flow,
+ eqr_width = render_params["EQR_WIDTH"],
+ eqr_height = render_params["EQR_HEIGHT"],
+ camera_rig_path = render_params["RIG_JSON_FILE"],
+ flow_algo = render_params["SIDE_FLOW_ALGORITHM"],
+ zero_parallax_dist = 10000,
+ interpupilary_dist = 6.4)
+
+ job = Job(columns = [left_chunk, right_chunk],
+ name = 'surround360_chunk_{:d}'.format(i))
+ jobs.append(job)
+
+ return db.run(jobs, force=True)
+
+
+def concat_stereo_panorama_chunks(db, chunks, render_params, is_left):
+ num_cams = 14
+ item_size = 10
+ print(chunks)
+ assert num_cams == len(chunks)
+
+ left_inputs = []
+ right_inputs = []
+ for c in range(num_cams):
+ left_chunk = db.ops.FrameInput()
+ right_chunk = db.ops.FrameInput()
+ left_inputs.append(left_chunk)
+ right_inputs.append(right_chunk)
+
+ args = db.protobufs.ConcatPanoramaChunksArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.final_eqr_width = render_params["FINAL_EQR_WIDTH"]
+ args.final_eqr_height = render_params["FINAL_EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ args.zero_parallax_dist = 10000
+ args.interpupilary_dist = 6.4
+ args.left = False
+ panorama = db.ops.ConcatPanoramaChunks(
+ *(left_inputs + right_inputs),
+ args=args)
+ output_op = db.ops.Output(columns=[panorama])
+
+ op_args = {
+ output_op: 'surround360_pano'
+ }
+ for c in range(num_cams):
+ op_args[left_inputs[c]] = chunks[c].column('left_chunk')
+ op_args[right_inputs[c]] = chunks[c].column('right_chunk')
+
+ job = Job(op_args=op_args)
+ bulk_job = BulkJob(output=output_op, jobs=[job])
+
+ tables = db.run(
+ bulk_job, force=True,
+ work_packet_size=item_size,
+ io_packet_size=item_size,
+ pipeline_instances_per_node=render_params["CORES"])
+ return tables[0]
+
+def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params,
+ start, end):
+ jobs = []
+ item_size = 10
+
+ proj_frames = []
+ for i in range(len(videos.tables())):
+ left_cam_idx = i
+ frame = videos.tables(left_cam_idx).as_op().range(start, end, task_size=item_size)
+ cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, task_size=item_size)
+
+ args = db.protobufs.ProjectSphericalArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ proj_frame = db.ops.ProjectSpherical(
+ frame = frame,
+ camera_id = cam_idx,
+ args=args)
+ proj_frames.append(proj_frame)
+
+ left_chunks = []
+ right_chunks = []
+ for i in range(len(videos.tables())):
+ left_cam_idx = i
+ right_cam_idx = (left_cam_idx + 1) % len(videos.tables())
+
+ left_proj_frame = proj_frames[left_cam_idx]
+ right_proj_frame = proj_frames[right_cam_idx]
+
+ left_flow, right_flow = db.ops.TemporalOpticalFlow(
+ left_projected_frame = left_proj_frame,
+ right_projected_frame = right_proj_frame,
+ flow_algo = render_params["SIDE_FLOW_ALGORITHM"],
+ camera_rig_path = render_params["RIG_JSON_FILE"])
+
+ left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk(
+ left_projected_frame = left_proj_frame,
+ left_flow = left_flow,
+ right_projected_frame = right_proj_frame,
+ right_flow = right_flow,
+ eqr_width = render_params["EQR_WIDTH"],
+ eqr_height = render_params["EQR_HEIGHT"],
+ camera_rig_path = render_params["RIG_JSON_FILE"],
+ flow_algo = render_params["SIDE_FLOW_ALGORITHM"],
+ zero_parallax_dist = 10000,
+ interpupilary_dist = 6.4)
+ left_chunks.append(left_chunk)
+ right_chunks.append(right_chunk)
+
+ jobs = []
+ args = db.protobufs.ConcatPanoramaChunksArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ args.zero_parallax_dist = 10000
+ args.interpupilary_dist = 6.4
+ args.left = True
+ panorama_left = db.ops.ConcatPanoramaChunks(*left_chunks, args=args)
+
+ args = db.protobufs.ConcatPanoramaChunksArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ args.zero_parallax_dist = 10000
+ args.interpupilary_dist = 6.4
+ args.left = False
+ panorama_right = db.ops.ConcatPanoramaChunks(*right_chunks, args=args)
+
+ job = Job(columns = [panorama_left, panorama_right],
+ name = 'surround360_pano_both')
+ return db.run(job, force=True)
+
+
+def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end):
+ warmup_size = 10
+ task_size = 100
+
+ left_frame = db.ops.FrameInput()
+ left_cam_idx = db.ops.Input()
+
+ right_frame = db.ops.FrameInput()
+ right_cam_idx = db.ops.Input()
+
+ args = db.protobufs.ProjectSphericalArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ left_proj_frame = db.ops.ProjectSpherical(
+ frame = left_frame,
+ camera_id = left_cam_idx,
+ args=args)
+ right_proj_frame = db.ops.ProjectSpherical(
+ frame = right_frame,
+ camera_id = right_cam_idx,
+ args=args)
+
+ left_flow, right_flow = db.ops.TemporalOpticalFlow(
+ left_projected_frame = left_proj_frame,
+ right_projected_frame = right_proj_frame,
+ flow_algo = render_params["SIDE_FLOW_ALGORITHM"],
+ camera_rig_path = render_params["RIG_JSON_FILE"],
+ warmup = warmup_size)
+
+ left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk(
+ left_projected_frame = left_proj_frame,
+ left_flow = left_flow,
+ right_projected_frame = right_proj_frame,
+ right_flow = right_flow,
+ eqr_width = render_params["EQR_WIDTH"],
+ eqr_height = render_params["EQR_HEIGHT"],
+ camera_rig_path = render_params["RIG_JSON_FILE"],
+ flow_algo = render_params["SIDE_FLOW_ALGORITHM"],
+ zero_parallax_dist = 10000,
+ interpupilary_dist = 6.4)
+
+ left_chunk_sample = left_chunk.sample()
+ right_chunk_sample = right_chunk.sample()
+
+ output_op = db.ops.Output(columns=[left_chunk_sample.lossless(),
+ right_chunk_sample.lossless()])
+
+ jobs = []
+ for i in range(len(videos.tables())):
+ left_idx = i
+ right_idx = (left_idx + 1) % len(videos.tables())
+
+ sample = db.sampler.range(start, end)
+
+ job = Job(op_args={
+ left_frame: videos.tables(left_idx).column('frame'),
+ left_cam_idx: videos_idx.tables(left_idx).column('camera_index'),
+ right_frame: videos.tables(right_idx).column('frame'),
+ right_cam_idx: videos_idx.tables(right_idx).column('camera_index'),
+ left_chunk_sample: sample,
+ right_chunk_sample: sample,
+ output_op: 'surround360_chunk_{:d}'.format(i),
+ })
+ jobs.append(job)
+
+ bulk_job = BulkJob(output=output_op, jobs=jobs)
+
+ return db.run(bulk_job, force=True,
+ work_packet_size=10,
+ io_packet_size=task_size,
+ pipeline_instances_per_node=render_params["CORES"])
+
+
+def fused_flow_and_stereo_chunk(db, overlap, render_params):
+ in_columns = ["index",
+ "projected_left", "frame_info_left",
+ "projected_right", "frame_info_right"]
+ input_op = db.ops.Input(in_columns)
+
+ args = db.protobufs.TemporalOpticalFlowArgs()
+ args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ temporal_op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])],
+ args=args)
+
+ args = db.protobufs.RenderStereoPanoramaChunkArgs()
+ args.eqr_width = render_params["EQR_WIDTH"]
+ args.eqr_height = render_params["EQR_HEIGHT"]
+ args.camera_rig_path = render_params["RIG_JSON_FILE"]
+ args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"]
+ args.zero_parallax_dist = 10000
+ args.interpupilary_dist = 6.4
+ op = db.ops.RenderStereoPanoramaChunk(
+ inputs=[(input_op, ["projected_left", "frame_info_left"]),
+ (temporal_op, ["flow_left", "frame_info_left"]),
+ (input_op, ["projected_right", "frame_info_right"]),
+ (temporal_op, ["flow_right", "frame_info_right"])],
+ args=args)
+
+ tasks = flow_tasks(db, overlap)
+
+ return db.run(tasks, op, 'surround360_chunk_fused', force=True)
+
+
+if __name__ == "__main__":
+ signal.signal(signal.SIGTERM, signal_term_handler)
+
+ parser = argparse.ArgumentParser(description='batch process video frames')
+ parser.add_argument('--root_dir', help='path to frame container dir', required=True)
+ parser.add_argument('--surround360_render_dir', help='project root path, containing bin and scripts dirs', required=False, default='.')
+ parser.add_argument('--start_frame', help='first frame index', required=True)
+ parser.add_argument('--end_frame', help='last frame index', required=True)
+ parser.add_argument('--quality', help='3k,4k,6k,8k', required=True)
+ parser.add_argument('--cubemap_width', help='default is to not generate cubemaps', required=False, default=0)
+ parser.add_argument('--cubemap_height', help='default is to not generate cubemaps', required=False, default=0)
+ parser.add_argument('--cubemap_format', help='photo,video', required=False, default='photo')
+ parser.add_argument('--save_debug_images', dest='save_debug_images', action='store_true')
+ parser.add_argument('--enable_top', dest='enable_top', action='store_true')
+ parser.add_argument('--enable_bottom', dest='enable_bottom', action='store_true')
+ parser.add_argument('--enable_pole_removal', dest='enable_pole_removal', action='store_true')
+ parser.add_argument('--resume', dest='resume', action='store_true', help='looks for a previous frame optical flow instead of starting fresh')
+ parser.add_argument('--rig_json_file', help='path to rig json file', required=True)
+ parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True)
+ parser.add_argument('--verbose', dest='verbose', action='store_true')
+ parser.add_argument('--cores', default=-1, type=int)
+ parser.set_defaults(save_debug_images=False)
+ parser.set_defaults(enable_top=False)
+ parser.set_defaults(enable_bottom=False)
+ parser.set_defaults(enable_pole_removal=False)
+ args = vars(parser.parse_args())
+
+ root_dir = args["root_dir"]
+ surround360_render_dir = args["surround360_render_dir"]
+ log_dir = root_dir + "/logs"
+ out_eqr_frames_dir = root_dir + "/eqr_frames"
+ out_cube_frames_dir = root_dir + "/cube_frames"
+ flow_dir = root_dir + "/flow"
+ debug_dir = root_dir + "/debug"
+ min_frame = int(args["start_frame"])
+ max_frame = int(args["end_frame"])
+ cubemap_width = int(args["cubemap_width"])
+ cubemap_height = int(args["cubemap_height"])
+ cubemap_format = args["cubemap_format"]
+ quality = args["quality"]
+ save_debug_images = args["save_debug_images"]
+ enable_top = args["enable_top"]
+ enable_bottom = args["enable_bottom"]
+ enable_pole_removal = args["enable_pole_removal"]
+ resume = args["resume"]
+ rig_json_file = args["rig_json_file"]
+ flow_alg = args["flow_alg"]
+ verbose = args["verbose"]
+
+ start_time = timer()
+
+ frame_range = range(min_frame, max_frame + 1)
+
+ render_params = {
+ "SURROUND360_RENDER_DIR": surround360_render_dir,
+ "LOG_DIR": log_dir,
+ "SRC_DIR": root_dir,
+ "PREV_FRAME_DIR": "NONE",
+ "OUT_EQR_DIR": out_eqr_frames_dir,
+ "OUT_CUBE_DIR": out_cube_frames_dir,
+ "CUBEMAP_WIDTH": cubemap_width,
+ "CUBEMAP_HEIGHT": cubemap_height,
+ "CUBEMAP_FORMAT": cubemap_format,
+ "RIG_JSON_FILE": rig_json_file,
+ "SIDE_FLOW_ALGORITHM": flow_alg,
+ "POLAR_FLOW_ALGORITHM": flow_alg,
+ "POLEREMOVAL_FLOW_ALGORITHM": flow_alg,
+ "EXTRA_FLAGS": "",
+ "CORES": args["cores"]
+ }
+
+
+ if save_debug_images:
+ render_params["EXTRA_FLAGS"] += " --save_debug_images"
+
+ if enable_top:
+ render_params["EXTRA_FLAGS"] += " --enable_top"
+
+ if enable_pole_removal and enable_bottom is False:
+ sys.stderr.write("Cannot use enable_pole_removal if enable_bottom is not used")
+ exit(1)
+
+ if enable_bottom:
+ render_params["EXTRA_FLAGS"] += " --enable_bottom"
+ if enable_pole_removal:
+ render_params["EXTRA_FLAGS"] += " --enable_pole_removal"
+ render_params["EXTRA_FLAGS"] += " --bottom_pole_masks_dir " + root_dir + "/pole_masks"
+
+ if quality == "3k":
+ render_params["SHARPENNING"] = 0.25
+ render_params["EQR_WIDTH"] = 3080
+ render_params["EQR_HEIGHT"] = 1540
+ render_params["FINAL_EQR_WIDTH"] = 3080
+ render_params["FINAL_EQR_HEIGHT"] = 3080
+ elif quality == "4k":
+ render_params["SHARPENNING"] = 0.25
+ render_params["EQR_WIDTH"] = 4200
+ render_params["EQR_HEIGHT"] = 1024
+ render_params["FINAL_EQR_WIDTH"] = 4096
+ render_params["FINAL_EQR_HEIGHT"] = 2048
+ elif quality == "6k":
+ render_params["SHARPENNING"] = 0.25
+ render_params["EQR_WIDTH"] = 6300
+ render_params["EQR_HEIGHT"] = 3072
+ render_params["FINAL_EQR_WIDTH"] = 6144
+ render_params["FINAL_EQR_HEIGHT"] = 6144
+ elif quality == "8k":
+ render_params["SHARPENNING"] = 0.25
+ render_params["EQR_WIDTH"] = 8400
+ render_params["EQR_HEIGHT"] = 4096
+ render_params["FINAL_EQR_WIDTH"] = 8192
+ render_params["FINAL_EQR_HEIGHT"] = 8192
+ else:
+ sys.stderr.write("Unrecognized quality setting: " + quality)
+ exit(1)
+
+ collection_name = 'surround360'
+ idx_collection_name = 'surround360_index'
+ db_start = timer()
+ with Database() as db:
+ print('DB', timer() - db_start)
+ db.load_op(
+ os.path.join(surround360_render_dir,
+ 'build/lib/libsurround360kernels.so'),
+ os.path.join(surround360_render_dir,
+ 'build/source/scanner_kernels/surround360_pb2.py'))
+
+ if not db.has_collection(collection_name):
+ print "----------- [Render] loading surround360 collection"
+ sys.stdout.flush()
+ paths = [os.path.join(root_dir, 'rgb', 'cam{:d}'.format(i), 'vid.mp4')
+ for i in range(1, 15)]
+ collection, _ = db.ingest_video_collection(collection_name, paths,
+ force=True)
+
+ idx_tables = []
+ num_rows = collection.tables(0).num_rows()
+ columns = ['camera_index']
+ for c in range(0, len(paths)):
+ rows = []
+ for i in range(num_rows):
+ rows.append([struct.pack('i', c)])
+ table_name = 'surround_cam_idx_{:d}'.format(c)
+ db.new_table(table_name, columns, rows, force=True)
+ idx_tables.append(db.table(table_name))
+ db.new_collection(idx_collection_name, idx_tables, force=True)
+
+
+ videos = db.collection(collection_name)
+ videos_idx = db.collection(idx_collection_name)
+
+ start_time = timer()
+ print "----------- [Render] processing frames ", min_frame, max_frame
+ sys.stdout.flush()
+ if verbose:
+ print(render_params)
+ sys.stdout.flush()
+
+ visualize = True
+ fused_4 = False # Fused project, flow, stereo chunk, and pano
+ fused_3 = True # Fused project, flow, and stereo chunk
+ fused_2 = False # Fused flow and stereo chunk
+ fused_1 = False # Nothing fused
+ if fused_4:
+ pano_col = fused_project_flow_chunk_concat(db, videos, videos_idx,
+ render_params,
+ min_frame, max_frame + 1)
+ pano_col.profiler().write_trace('fused4.trace')
+ elif fused_3:
+ flow_stereo_start = timer()
+ chunk_col = fused_project_flow_and_stereo_chunk(
+ db, videos, videos_idx, render_params, min_frame, max_frame + 1)
+ print(db.summarize())
+ chunk_col = [db.table('surround360_chunk_{:d}'.format(i))
+ for i in range(14)]
+ print('Proj flow stereo', timer() - flow_stereo_start)
+ concat_start = timer()
+ pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params,
+ True)
+ print('Concat', timer() - concat_start)
+ chunk_col[0].profiler().write_trace('fused3.trace')
+ pano_col.profiler().write_trace('fused3_concat.trace')
+ elif fused_2:
+ proj_start = timer()
+ proj_col = project_images(db, videos, videos_idx, render_params)
+ print('Proj', timer() - proj_start)
+ flow_stereo_start = timer()
+ chunk_col = fused_flow_and_stereo_chunk(db, proj_col, render_params)
+ print('Flow stereo', timer() - flow_stereo_start)
+ concat_start = timer()
+ pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params,
+ True)
+ print('Concat', timer() - concat_start)
+ elif fused_1:
+ flow_col = compute_temporal_flow(db, proj_col, render_params)
+ if save_debug_images:
+ t1 = flow_col.tables(0)
+ for fi, tup in t1.load(['flow_left', 'frame_info_left']):
+ frame_info = db.protobufs.FrameInfo()
+ frame_info.ParseFromString(tup[1])
+
+ frame = np.frombuffer(tup[0], dtype=np.float32).reshape(
+ frame_info.height,
+ frame_info.width,
+ 2)
+ frame = np.append(frame, np.zeros((frame_info.height, frame_info.width, 1)), axis=2)
+ scipy.misc.toimage(frame[:,:,:]).save('flow_test.png')
+
+ chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params)
+
+ png_start = timer()
+ pano_col = db.table('surround360_pano')
+ # left_table = pano_col[0]
+ # right_table = pano_col[1]
+ if visualize:
+ pano_col.column('panorama').save_mp4('pano_test',
+ scale=(2048, 2048))
+ print('To png', timer() - png_start)
+
+ end_time = timer()
+
+ if verbose:
+ total_runtime = end_time - start_time
+ avg_runtime = total_runtime / float(max_frame - min_frame + 1)
+ print "Render total runtime:", total_runtime, "sec"
+ print "Average runtime:", avg_runtime, "sec/frame"
+ sys.stdout.flush()
diff --git a/surround360_render/source/calibration/GeometricCalibration.cpp b/surround360_render/source/calibration/GeometricCalibration.cpp
index a669c8bf..6c202090 100644
--- a/surround360_render/source/calibration/GeometricCalibration.cpp
+++ b/surround360_render/source/calibration/GeometricCalibration.cpp
@@ -9,6 +9,7 @@
#include "opencv2/stitching/detail/matchers.hpp"
#include "ceres/ceres.h"
#include "ceres/rotation.h"
+#include "ceres/types.h"
#include "GeometricCalibration.h"
#include "Camera.h"
@@ -40,6 +41,7 @@ DEFINE_double(perturb_principals, 0, "pertub principals (pixels)");
DEFINE_int64(experiments, 1, "calibrate multiple times");
DEFINE_bool(discard_outside_fov, true, "discard matches outside fov");
DEFINE_bool(save_debug_images, false, "save intermediate images");
+//#define FLAGS_save_debug_images false
// is the stem expected to be the camera id? i.e. the path is of the form:
// / ... /.
@@ -767,6 +769,10 @@ void solve(
std::vector& rotations,
std::vector& traces) {
ceres::Solver::Options options;
+ options.linear_solver_type = ceres::ITERATIVE_SCHUR;
+ options.preconditioner_type = ceres::SCHUR_JACOBI;
+ options.num_threads = 32;
+ options.num_linear_solver_threads = 32;
options.use_inner_iterations = true;
options.max_num_iterations = 500;
options.minimizer_progress_to_stdout = false;
@@ -804,13 +810,13 @@ void refine(
triangulateTraces(traces, keypointMap, cameras);
// visualization for debugging
- showMatches(
- cameras,
- keypointMap,
- overlaps,
- traces,
- debugDir,
- pass);
+ // showMatches(
+ // cameras,
+ // keypointMap,
+ // overlaps,
+ // traces,
+ // debugDir,
+ // pass);
// read camera parameters from cameras
std::vector positions;
diff --git a/surround360_render/source/camera_isp/Raw2Rgb.cpp b/surround360_render/source/camera_isp/Raw2Rgb.cpp
index 594447b2..551591f3 100644
--- a/surround360_render/source/camera_isp/Raw2Rgb.cpp
+++ b/surround360_render/source/camera_isp/Raw2Rgb.cpp
@@ -403,6 +403,8 @@ int main(int argc, char* argv[]) {
FLAGS_input_image_path,
CV_LOAD_IMAGE_GRAYSCALE | CV_LOAD_IMAGE_ANYDEPTH);
+ LOG(INFO) << "Camera path " << FLAGS_input_image_path;
+
if (inputImage.cols > 2 && inputImage.rows > 2) {
const uint8_t depth = inputImage.type() & CV_MAT_DEPTH_MASK;
diff --git a/surround360_render/source/optical_flow/NovelView.h b/surround360_render/source/optical_flow/NovelView.h
index eb6f97e5..0eb435e6 100644
--- a/surround360_render/source/optical_flow/NovelView.h
+++ b/surround360_render/source/optical_flow/NovelView.h
@@ -133,8 +133,17 @@ class NovelViewGenerator {
const LazyNovelViewBuffer& lazyBuffer) = 0;
// for debugging
+ virtual Mat getImageL() { return Mat(); }
+ virtual Mat getImageR() { return Mat(); }
+
+ virtual void setImageL(const Mat& image) = 0;
+ virtual void setImageR(const Mat& image) = 0;
+
virtual Mat getFlowLtoR() { return Mat(); }
virtual Mat getFlowRtoL() { return Mat(); }
+
+ virtual void setFlowLtoR(const Mat& flow) = 0;
+ virtual void setFlowRtoL(const Mat& flow) = 0;
};
// this is a base class for novel view generators that work by reduction to optical flow.
@@ -162,8 +171,17 @@ class NovelViewGeneratorLazyFlow : public NovelViewGenerator {
pair combineLazyNovelViews(const LazyNovelViewBuffer& lazyBuffer);
+ Mat getImageL() { return imageL; }
+ Mat getImageR() { return imageR; }
+
+ void setImageL(const Mat& image) { imageL = image; }
+ void setImageR(const Mat& image) { imageR = image; }
+
Mat getFlowLtoR() { return flowLtoR; }
Mat getFlowRtoL() { return flowRtoL; }
+
+ void setFlowLtoR(const Mat& flow) { flowLtoR = flow; }
+ void setFlowRtoL(const Mat& flow) { flowRtoL = flow; }
};
// the name "asymmetric" here refers to the idea that we compute an optical flow from
diff --git a/surround360_render/source/render/RigDescription.cpp b/surround360_render/source/render/RigDescription.cpp
index 8864906e..f68162c3 100644
--- a/surround360_render/source/render/RigDescription.cpp
+++ b/surround360_render/source/render/RigDescription.cpp
@@ -15,8 +15,11 @@ using namespace cv;
using namespace std;
using namespace surround360::util;
-RigDescription::RigDescription(const string& filename) {
- rig = Camera::loadRig(filename);
+RigDescription::RigDescription(const string& filename)
+ : RigDescription(Camera::loadRig(filename)) {
+}
+
+RigDescription::RigDescription(const Camera::Rig& rig) {
for (const Camera& camera : rig) {
if (camera.group.find("side") != string::npos) {
rigSideOnly.emplace_back(camera);
diff --git a/surround360_render/source/render/RigDescription.h b/surround360_render/source/render/RigDescription.h
index a452e188..ca32a3ff 100644
--- a/surround360_render/source/render/RigDescription.h
+++ b/surround360_render/source/render/RigDescription.h
@@ -29,6 +29,8 @@ struct RigDescription {
Camera::Rig rigSideOnly;
RigDescription(const string& filename);
+ RigDescription(const Camera::Rig& rig);
+
// find the camera that is closest to pointing in the provided direction
// ignore those with excessive distance from the camera axis to the rig center
const Camera& findCameraByDirection(
@@ -60,4 +62,29 @@ struct RigDescription {
}
};
+// measured in radians from forward
+inline float approximateFov(const Camera& camera, const bool vertical) {
+ Camera::Vector2 a = camera.principal;
+ Camera::Vector2 b = camera.principal;
+ if (vertical) {
+ a.y() = 0;
+ b.y() = camera.resolution.y();
+ } else {
+ a.x() = 0;
+ b.x() = camera.resolution.x();
+ }
+ return acos(max(
+ camera.rig(a).direction().dot(camera.forward()),
+ camera.rig(b).direction().dot(camera.forward())));
+}
+
+// measured in radians from forward
+inline float approximateFov(const Camera::Rig& rig, const bool vertical) {
+ float result = 0;
+ for (const auto& camera : rig) {
+ result = std::max(result, approximateFov(camera, vertical));
+ }
+ return result;
+}
+
} // namespace surround360
diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp
new file mode 100644
index 00000000..b0d24c08
--- /dev/null
+++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp
@@ -0,0 +1,147 @@
+#include "render/RigDescription.h"
+#include "optical_flow/NovelView.h"
+#include "util/MathUtil.h"
+#include "source/scanner_kernels/surround360.pb.h"
+
+#include "scanner/api/kernel.h"
+#include "scanner/api/op.h"
+#include "scanner/util/memory.h"
+#include "scanner/util/opencv.h"
+
+using namespace scanner;
+
+namespace surround360 {
+namespace {
+void padToheight(Mat& unpaddedImage, const int targetHeight) {
+ const int paddingAbove = (targetHeight - unpaddedImage.rows) / 2;
+ const int paddingBelow = targetHeight - unpaddedImage.rows - paddingAbove;
+ cv::copyMakeBorder(
+ unpaddedImage,
+ unpaddedImage,
+ paddingAbove,
+ paddingBelow,
+ 0,
+ 0,
+ BORDER_CONSTANT,
+ Scalar(0.0, 0.0, 0.0));
+}
+}
+
+using namespace optical_flow;
+using namespace math_util;
+
+class ConcatPanoramaChunksKernelCPU : public BatchedKernel, public VideoKernel {
+ public:
+ ConcatPanoramaChunksKernelCPU(const KernelConfig& config)
+ : BatchedKernel(config),
+ device_(config.devices[0]) {
+ args_.ParseFromArray(config.args.data(), config.args.size());
+
+ rig_.reset(new RigDescription(args_.camera_rig_path()));
+
+ num_chunks_ = config.input_columns.size() / 2;
+ }
+
+ void new_frame_info() override {
+ const int numCams = 14;
+ const float cameraRingRadius = rig_->getRingRadius();
+ const float camFovHorizontalDegrees =
+ 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI);
+ const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees);
+ const float overlapAngleDegrees =
+ (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams);
+ const int camImageWidth = frame_info_.width();
+ const int camImageHeight = frame_info_.height();
+ const int overlapImageWidth =
+ float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees);
+
+ const float v =
+ atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f));
+ const float psi =
+ asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius);
+ const float vergeAtInfinitySlabDisplacement =
+ psi * (float(camImageWidth) / fovHorizontalRadians);
+ const float theta = -M_PI / 2.0f + v + psi;
+ zeroParallaxNovelViewShiftPixels_ =
+ float(args_.eqr_width()) * (theta / (2.0f * M_PI));
+ if (!args_.left()) {
+ zeroParallaxNovelViewShiftPixels_ *= -1;
+ }
+ }
+
+ void execute(const BatchedColumns& input_columns,
+ BatchedColumns& output_columns) override {
+ check_frame(device_, input_columns[0][0]);
+
+ i32 input_count = (i32)num_rows(input_columns[0]);
+ size_t output_image_width = (size_t)args_.final_eqr_width();
+ size_t output_image_height = (size_t)args_.final_eqr_height();
+ output_image_width += (output_image_width % 2);
+ output_image_height += (output_image_height % 2);
+ size_t output_image_size =
+ output_image_width * output_image_height * 3;
+ FrameInfo info(output_image_height, output_image_width, 3, FrameType::U8);
+ std::vector output_frames = new_frames(device_, info, input_count);
+
+ std::vector left_pano_chunks(num_chunks_, Mat());
+ std::vector right_pano_chunks(num_chunks_, Mat());
+ cv::Mat left_pano;
+ cv::Mat right_pano;
+ for (i32 i = 0; i < input_count; ++i) {
+ for (i32 c = 0; c < num_chunks_; ++c) {
+ auto &left_chunk_col = input_columns[c];
+ cv::cvtColor(frame_to_mat(left_chunk_col[i].as_const_frame()),
+ left_pano_chunks[c], CV_BGRA2BGR);
+
+ auto &right_chunk_col = input_columns[num_chunks_ + c];
+ cv::cvtColor(frame_to_mat(right_chunk_col[i].as_const_frame()),
+ right_pano_chunks[c], CV_BGRA2BGR);
+ }
+ left_pano = stackHorizontal(left_pano_chunks);
+ left_pano =
+ offsetHorizontalWrap(left_pano, zeroParallaxNovelViewShiftPixels_);
+
+ right_pano = stackHorizontal(right_pano_chunks);
+ right_pano =
+ offsetHorizontalWrap(right_pano, -zeroParallaxNovelViewShiftPixels_);
+
+ padToheight(left_pano, args_.eqr_height());
+ padToheight(right_pano, args_.eqr_height());
+
+ resize(left_pano, left_pano,
+ Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0,
+ INTER_CUBIC);
+ resize(right_pano, right_pano,
+ Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0,
+ INTER_CUBIC);
+
+ cv::Mat stereo_equirect =
+ stackVertical(vector({left_pano, right_pano}));
+
+ u8 *output = output_frames[i]->data;
+ for (i32 r = 0; r < stereo_equirect.rows; ++r) {
+ memcpy(output + r * output_image_width * sizeof(char) * 3,
+ stereo_equirect.data + stereo_equirect.step * r,
+ stereo_equirect.cols * sizeof(char) * 3);
+ }
+
+ insert_frame(output_columns[0], output_frames[i]);
+ }
+ }
+
+ private:
+ surround360::proto::ConcatPanoramaChunksArgs args_;
+ std::unique_ptr rig_;
+ DeviceHandle device_;
+ int num_chunks_;
+ float zeroParallaxNovelViewShiftPixels_;
+};
+
+REGISTER_OP(ConcatPanoramaChunks)
+ .variadic_inputs()
+ .frame_output("panorama");
+
+REGISTER_KERNEL(ConcatPanoramaChunks, ConcatPanoramaChunksKernelCPU)
+ .device(DeviceType::CPU)
+ .num_devices(1);
+}
diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp
new file mode 100644
index 00000000..e7ea48ea
--- /dev/null
+++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp
@@ -0,0 +1,102 @@
+#include "render/RigDescription.h"
+#include "render/ImageWarper.h"
+#include "source/scanner_kernels/surround360.pb.h"
+
+#include "scanner/api/kernel.h"
+#include "scanner/api/op.h"
+#include "scanner/util/memory.h"
+#include "scanner/util/opencv.h"
+
+#include
+
+using namespace scanner;
+
+namespace surround360 {
+
+class ProjectSphericalKernelCPU : public BatchedKernel, public VideoKernel {
+ public:
+ ProjectSphericalKernelCPU(const KernelConfig& config)
+ : BatchedKernel(config),
+ device_(config.devices[0]) {
+ args_.ParseFromArray(config.args.data(), config.args.size());
+
+ // Initialize camera rig
+ rig_.reset(new RigDescription(args_.camera_rig_path()));
+
+ hRadians_ = 2 * approximateFov(rig_->rigSideOnly, false);
+ vRadians_ = 2 * approximateFov(rig_->rigSideOnly, true);
+ }
+
+ void reset() override {
+ is_reset_ = true;
+ }
+
+ void execute(const BatchedColumns& input_columns,
+ BatchedColumns& output_columns) override {
+ auto& frame_col = input_columns[0];
+ auto& camera_id_col = input_columns[1];
+ check_frame(device_, frame_col[0]);
+
+ if (is_reset_) {
+ // Use the new camera id to update the spherical projection parameters
+ is_reset_ = false;
+
+ camIdx_ = *((int*)camera_id_col[0].buffer);
+ const Camera& camera = rig_->rigSideOnly[camIdx_];
+
+ // the negative sign here is so the camera array goes clockwise
+ const int numCameras = 14;
+ float direction = -float(camIdx_) / float(numCameras) * 2.0f * M_PI;
+ leftAngle_ = direction + hRadians_ / 2;
+ rightAngle_ = direction - hRadians_ / 2;
+ topAngle_ = vRadians_ / 2;
+ bottomAngle_ = -vRadians_ / 2;
+ }
+
+ i32 input_count = (i32)num_rows(frame_col);
+ size_t output_image_width = args_.eqr_width() * hRadians_ / (2 * M_PI);
+ size_t output_image_height = args_.eqr_height() * vRadians_ / M_PI;
+ size_t output_image_size = output_image_width * output_image_height * 4;
+ FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8);
+ std::vector output_frames = new_frames(device_, info, input_count);
+
+ for (i32 i = 0; i < input_count; ++i) {
+ cv::Mat input = frame_to_mat(frame_col[i].as_const_frame());
+ cv::Mat tmp;
+ cv::cvtColor(input, tmp, CV_BGR2BGRA);
+
+ cv::Mat projection_image = frame_to_mat(output_frames[i]);
+
+ surround360::warper::bicubicRemapToSpherical(
+ projection_image, tmp, rig_->rigSideOnly[camIdx_], leftAngle_,
+ rightAngle_, topAngle_, bottomAngle_);
+
+ insert_frame(output_columns[0], output_frames[i]);
+ }
+ }
+
+ private:
+ surround360::proto::ProjectSphericalArgs args_;
+ std::unique_ptr rig_;
+ DeviceHandle device_;
+ bool is_reset_ = true;
+
+ float hRadians_;
+ float vRadians_;
+
+ int camIdx_;
+ float leftAngle_;
+ float rightAngle_;
+ float topAngle_;
+ float bottomAngle_;
+};
+
+REGISTER_OP(ProjectSpherical)
+ .frame_input("frame")
+ .input("camera_id")
+ .frame_output("projected_frame");
+
+REGISTER_KERNEL(ProjectSpherical, ProjectSphericalKernelCPU)
+ .device(DeviceType::CPU)
+ .num_devices(1);
+}
diff --git a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp
new file mode 100644
index 00000000..c95b81ca
--- /dev/null
+++ b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp
@@ -0,0 +1,156 @@
+#include "render/RigDescription.h"
+#include "optical_flow/NovelView.h"
+#include "util/MathUtil.h"
+#include "source/scanner_kernels/surround360.pb.h"
+
+#include "scanner/api/kernel.h"
+#include "scanner/api/op.h"
+#include "scanner/util/memory.h"
+#include "scanner/util/opencv.h"
+
+using namespace scanner;
+
+namespace surround360 {
+
+using namespace optical_flow;
+using namespace math_util;
+
+class RenderStereoPanoramaChunkKernelCPU : public BatchedKernel,
+ public VideoKernel {
+public:
+ RenderStereoPanoramaChunkKernelCPU(const KernelConfig &config)
+ : BatchedKernel(config), device_(config.devices[0]) {
+ args_.ParseFromArray(config.args.data(), config.args.size());
+
+ rig_.reset(new RigDescription(args_.camera_rig_path()));
+
+ overlap_image_width_ = -1;
+ novel_view_gen_.reset(
+ new NovelViewGeneratorAsymmetricFlow(args_.flow_algo()));
+ }
+
+ void new_frame_info() override {
+ const int numCams = 14;
+ const float cameraRingRadius = rig_->getRingRadius();
+ const float camFovHorizontalDegrees =
+ 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI);
+ fov_horizontal_radians_ = toRadians(camFovHorizontalDegrees);
+ const float overlapAngleDegrees =
+ (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams);
+ const int camImageWidth = frame_info_.width();
+ const int camImageHeight = frame_info_.height();
+ overlap_image_width_ = float(camImageWidth) *
+ (overlapAngleDegrees / camFovHorizontalDegrees);
+ num_novel_views_ =
+ camImageWidth - overlap_image_width_; // per image pair
+
+ const float v =
+ atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f));
+ const float psi =
+ asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius);
+ const float vergeAtInfinitySlabDisplacement =
+ psi * (float(camImageWidth) / fov_horizontal_radians_);
+ const float theta = -M_PI / 2.0f + v + psi;
+ const float zeroParallaxNovelViewShiftPixels =
+ float(args_.eqr_width()) * (theta / (2.0f * M_PI));
+
+ int currChunkX = 0;
+ lazy_view_buffer_.reset(new LazyNovelViewBuffer(args_.eqr_width() / numCams,
+ camImageHeight));
+ for (int nvIdx = 0; nvIdx < num_novel_views_; ++nvIdx) {
+ const float shift = float(nvIdx) / float(num_novel_views_);
+ const float slabShift =
+ float(camImageWidth) * 0.5f - float(num_novel_views_ - nvIdx);
+
+ for (int v = 0; v < camImageHeight; ++v) {
+ lazy_view_buffer_->warpL[currChunkX][v] =
+ Point3f(slabShift + vergeAtInfinitySlabDisplacement, v, shift);
+ lazy_view_buffer_->warpR[currChunkX][v] =
+ Point3f(slabShift - vergeAtInfinitySlabDisplacement, v, shift);
+ }
+ ++currChunkX;
+ }
+ }
+
+ void execute(const BatchedColumns& input_columns,
+ BatchedColumns& output_columns) override {
+ auto& left_frame_col = input_columns[0];
+ auto& left_flow_col = input_columns[1];
+ auto& right_frame_col = input_columns[2];
+ auto& right_flow_col = input_columns[3];
+ check_frame(device_, left_frame_col[0]);
+ assert(overlap_image_width_ != -1);
+
+ i32 input_count = (i32)num_rows(left_frame_col);
+ size_t output_image_width = num_novel_views_;
+ size_t output_image_height = frame_info_.height();
+ size_t output_image_size =
+ output_image_width * output_image_height * 4;
+ FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8);
+ std::vector frames = new_frames(device_, info, input_count * 2);
+
+ for (i32 i = 0; i < input_count; ++i) {
+ cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame());
+ cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame());
+ cv::Mat left_overlap_input =
+ left_input(cv::Rect(left_input.cols - overlap_image_width_, 0,
+ overlap_image_width_, left_input.rows));
+ cv::Mat right_overlap_input =
+ right_input(cv::Rect(0, 0,
+ overlap_image_width_, right_input.rows));
+
+
+ cv::Mat left_flow = frame_to_mat(left_flow_col[i].as_const_frame());
+ cv::Mat right_flow = frame_to_mat(right_flow_col[i].as_const_frame());
+
+ // Initialize NovelViewGenerator with images and flow since we are
+ // bypassing the prepare method
+ novel_view_gen_->setImageL(left_overlap_input);
+ novel_view_gen_->setImageR(right_overlap_input);
+ novel_view_gen_->setFlowLtoR(left_flow);
+ novel_view_gen_->setFlowRtoL(right_flow);
+ std::pair lazyNovelChunksLR =
+ novel_view_gen_->combineLazyNovelViews(*lazy_view_buffer_.get());
+ const cv::Mat& chunkL = lazyNovelChunksLR.first;
+ const cv::Mat& chunkR = lazyNovelChunksLR.second;
+
+ u8* left_output = frames[2 * i]->data;
+ u8* right_output = frames[2 * i + 1]->data;
+ for (i32 r = 0; r < chunkL.rows; ++r) {
+ memcpy(left_output + r * output_image_width * sizeof(char) * 4,
+ chunkL.data + chunkL.step * r,
+ chunkL.cols * sizeof(char) * 4);
+ memcpy(right_output + r * output_image_width * sizeof(char) * 4,
+ chunkR.data + chunkR.step * r,
+ chunkR.cols * sizeof(char) * 4);
+ }
+
+ insert_frame(output_columns[0], frames[2 * i]);
+ insert_frame(output_columns[1], frames[2 * i + 1]);
+ }
+ }
+
+ private:
+ surround360::proto::RenderStereoPanoramaChunkArgs args_;
+ std::unique_ptr rig_;
+ DeviceHandle device_;
+ float fov_horizontal_radians_;
+ int overlap_image_width_;
+ int num_novel_views_;
+
+ std::unique_ptr novel_view_gen_;
+ std::unique_ptr lazy_view_buffer_;
+};
+
+REGISTER_OP(RenderStereoPanoramaChunk)
+ .frame_input("left_projected_frame")
+ .frame_input("left_flow")
+ .frame_input("right_projected_frame")
+ .frame_input("right_flow")
+ .frame_output("left_chunk")
+ .frame_output("right_chunk");
+
+REGISTER_KERNEL(RenderStereoPanoramaChunk, RenderStereoPanoramaChunkKernelCPU)
+ .device(DeviceType::CPU)
+ .num_devices(1);
+}
diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto
new file mode 100644
index 00000000..1afc20fb
--- /dev/null
+++ b/surround360_render/source/scanner_kernels/surround360.proto
@@ -0,0 +1,34 @@
+syntax = "proto3";
+
+package surround360.proto;
+
+message ProjectSphericalArgs {
+ int32 eqr_height = 1;
+ int32 eqr_width = 2;
+ string camera_rig_path = 3;
+}
+
+message TemporalOpticalFlowArgs {
+ string flow_algo = 1;
+ string camera_rig_path = 2;
+}
+
+message RenderStereoPanoramaChunkArgs {
+ int32 eqr_height = 1;
+ int32 eqr_width = 2;
+ string camera_rig_path = 3;
+ string flow_algo = 4;
+ float zero_parallax_dist = 5;
+ float interpupilary_dist = 6;
+}
+
+message ConcatPanoramaChunksArgs {
+ int32 eqr_height = 1;
+ int32 eqr_width = 2;
+ int32 final_eqr_height = 7;
+ int32 final_eqr_width = 8;
+ string camera_rig_path = 3;
+ float zero_parallax_dist = 4;
+ float interpupilary_dist = 5;
+ bool left = 6;
+}
diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp
new file mode 100644
index 00000000..160d3cd3
--- /dev/null
+++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp
@@ -0,0 +1,129 @@
+#include "render/RigDescription.h"
+#include "optical_flow/NovelView.h"
+#include "util/MathUtil.h"
+#include "source/scanner_kernels/surround360.pb.h"
+
+#include "scanner/api/kernel.h"
+#include "scanner/api/op.h"
+#include "scanner/util/memory.h"
+#include "scanner/util/opencv.h"
+
+using namespace scanner;
+
+namespace surround360 {
+
+using namespace optical_flow;
+using namespace math_util;
+
+class TemporalOpticalFlowKernelCPU : public BatchedKernel, public VideoKernel {
+ public:
+ TemporalOpticalFlowKernelCPU(const KernelConfig& config)
+ : BatchedKernel(config),
+ device_(config.devices[0]) {
+ args_.ParseFromArray(config.args.data(), config.args.size());
+
+ rig_.reset(new RigDescription(args_.camera_rig_path()));
+
+ overlap_image_width_ = -1;
+ novel_view_gen_.reset(
+ new NovelViewGeneratorAsymmetricFlow(args_.flow_algo()));
+ }
+
+ void reset() override {
+ prev_frame_flow_l_to_r_ = cv::Mat();
+ prev_frame_flow_r_to_l_ = cv::Mat();
+ prev_overlap_image_l_ = cv::Mat();
+ prev_overlap_image_r_ = cv::Mat();
+ }
+
+ void new_frame_info() override {
+ const int numCams = 14;
+ const float cameraRingRadius = rig_->getRingRadius();
+ const float camFovHorizontalDegrees =
+ 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI);
+ const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees);
+ const float overlapAngleDegrees =
+ (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams);
+ const int camImageWidth = frame_info_.width();
+ const int camImageHeight = frame_info_.height();
+ overlap_image_width_ =
+ float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees);
+ }
+
+ void execute(const BatchedColumns& input_columns,
+ BatchedColumns& output_columns) override {
+ auto& left_frame_col = input_columns[0];
+ auto& right_frame_col = input_columns[1];
+ check_frame(device_, left_frame_col[0]);
+ assert(overlap_image_width_ != -1);
+
+ i32 input_count = (i32)num_rows(left_frame_col);
+ size_t output_image_width = overlap_image_width_;
+ size_t output_image_height = frame_info_.height();
+ size_t output_image_size =
+ output_image_width * output_image_height * 2 * sizeof(float);
+ FrameInfo info(output_image_height, output_image_width, 2, FrameType::F32);
+ std::vector frames = new_frames(device_, info, input_count * 2);
+
+ for (i32 i = 0; i < input_count; ++i) {
+ cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame());
+ cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame());
+
+ cv::Mat left_overlap_input =
+ left_input(cv::Rect(left_input.cols - overlap_image_width_, 0,
+ overlap_image_width_, left_input.rows));
+ cv::Mat right_overlap_input =
+ right_input(cv::Rect(0, 0,
+ overlap_image_width_, right_input.rows));
+
+ novel_view_gen_->prepare(left_overlap_input, right_overlap_input,
+ prev_frame_flow_l_to_r_, prev_frame_flow_r_to_l_,
+ prev_overlap_image_l_, prev_overlap_image_r_);
+
+ left_overlap_input.copyTo(prev_overlap_image_l_);
+ right_overlap_input.copyTo(prev_overlap_image_r_);
+ prev_frame_flow_l_to_r_ = novel_view_gen_->getFlowLtoR();
+ prev_frame_flow_r_to_l_ = novel_view_gen_->getFlowRtoL();
+
+ u8* left_output = frames[2 * i]->data;
+ u8* right_output = frames[2 * i + 1]->data;
+ const auto& left_flow = novel_view_gen_->getFlowLtoR();
+ const auto& right_flow = novel_view_gen_->getFlowRtoL();
+ for (i32 r = 0; r < left_flow.rows; ++r) {
+ memcpy(left_output + r * output_image_width * sizeof(float) * 2,
+ left_flow.data + left_flow.step * r,
+ left_flow.cols * sizeof(float) * 2);
+ memcpy(right_output + r * output_image_width * sizeof(float) * 2,
+ right_flow.data + right_flow.step * r,
+ right_flow.cols * sizeof(float) * 2);
+ }
+
+ insert_frame(output_columns[0], frames[2 * i]);
+ insert_frame(output_columns[1], frames[2 * i + 1]);
+ }
+ }
+
+ private:
+ surround360::proto::TemporalOpticalFlowArgs args_;
+ std::unique_ptr rig_;
+ DeviceHandle device_;
+ int overlap_image_width_;
+
+ std::unique_ptr novel_view_gen_;
+ cv::Mat prev_frame_flow_l_to_r_;
+ cv::Mat prev_frame_flow_r_to_l_;
+ cv::Mat prev_overlap_image_l_;
+ cv::Mat prev_overlap_image_r_;
+};
+
+REGISTER_OP(TemporalOpticalFlow)
+ .frame_input("left_projected_frame")
+ .frame_input("right_projected_frame")
+ .frame_output("left_flow")
+ .frame_output("right_flow")
+ .bounded_state();
+
+REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU)
+ .device(DeviceType::CPU)
+ .num_devices(1);
+}
diff --git a/surround360_render/source/test/TestRenderStereoPanorama.cpp b/surround360_render/source/test/TestRenderStereoPanorama.cpp
index 0f302feb..f6b5f669 100644
--- a/surround360_render/source/test/TestRenderStereoPanorama.cpp
+++ b/surround360_render/source/test/TestRenderStereoPanorama.cpp
@@ -71,31 +71,6 @@ DEFINE_string(cubemap_format, "video", "either video or photo
const Camera::Vector3 kGlobalUp = Camera::Vector3::UnitZ();
-// measured in radians from forward
-float approximateFov(const Camera& camera, const bool vertical) {
- Camera::Vector2 a = camera.principal;
- Camera::Vector2 b = camera.principal;
- if (vertical) {
- a.y() = 0;
- b.y() = camera.resolution.y();
- } else {
- a.x() = 0;
- b.x() = camera.resolution.x();
- }
- return acos(max(
- camera.rig(a).direction().dot(camera.forward()),
- camera.rig(b).direction().dot(camera.forward())));
-}
-
-// measured in radians from forward
-float approximateFov(const Camera::Rig& rig, const bool vertical) {
- float result = 0;
- for (const auto& camera : rig) {
- result = std::max(result, approximateFov(camera, vertical));
- }
- return result;
-}
-
void projectSideToSpherical(
Mat& dst,
const Mat& src,