diff --git a/surround360_render/CMakeLists.txt b/surround360_render/CMakeLists.txt index 1ee7c391..0b2e0d52 100644 --- a/surround360_render/CMakeLists.txt +++ b/surround360_render/CMakeLists.txt @@ -2,9 +2,13 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.2) PROJECT(Surround360Render CXX) +OPTION(USE_SCANNER "Enable using Scanner for distributed rendering" ON) + FIND_PACKAGE(OpenCV) FIND_PACKAGE(Ceres REQUIRED) +FIND_PACKAGE(Boost REQUIRED) +INCLUDE_DIRECTORIES(${Boost_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${OpenCV_INCLUDE_DIRS}) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/util) @@ -15,6 +19,8 @@ INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/source/render) INCLUDE_DIRECTORIES(${CMAKE_BINARY_DIR}) INCLUDE_DIRECTORIES(/usr/local/include/eigen3) +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC") + if (CMAKE_CXX_COMPILER_ID MATCHES "Clang") # required for clang, including AppleClang LINK_DIRECTORIES(/usr/local/lib) @@ -53,7 +59,7 @@ ADD_CUSTOM_TARGET( _source SOURCES ${SRC} ) ### Halide support IF (DEFINED HALIDE_DIR) - INCLUDE("${HALIDE_DIR}/../HalideGenerator.cmake") + INCLUDE("${HALIDE_DIR}/HalideGenerator.cmake") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -DUSE_HALIDE") @@ -90,14 +96,19 @@ FILE(GLOB optical_flow_SRC "source/optical_flow/*.cpp") FILE(GLOB render_SRC "source/render/*.cpp") FILE(GLOB util_SRC "source/util/*.cpp") -ADD_LIBRARY( - LibVrCamera +ADD_LIBRARY(LibVrCamera source/util/SystemUtil.cpp ${calibration_SRC} ${optical_flow_SRC} ${render_SRC} ${util_SRC} ) +# TARGET_LINK_LIBRARIES(LibVrCamera +# folly +# ceres +# LibJSON +# ${OpenCV_LIBS} +# ${PLATFORM_SPECIFIC_LIBS}) TARGET_COMPILE_FEATURES(LibVrCamera PRIVATE cxx_range_for) ### Raw2Rgb ### @@ -198,6 +209,12 @@ TARGET_LINK_LIBRARIES( LibJSON glog gflags + +# folly +# ceres +# LibJSON + ${OpenCV_LIBS} + ${PLATFORM_SPECIFIC_LIBS} ) ### TestOpticalFlow ### @@ -498,11 +515,31 @@ TARGET_LINK_LIBRARIES( LibJSON gflags glog - folly + -L${Boost_LIBRARY_DIRS} boost_filesystem boost_system double-conversion + folly + ${CERES_LIBRARIES} ${OpenCV_LIBS} ${PLATFORM_SPECIFIC_LIBS} - ${CERES_LIBRARIES} ) + + +IF (USE_SCANNER) + SET(SCANNER_PATH "~/repos/scanner/") + include(${SCANNER_PATH}/cmake/Util/Op.cmake) + build_op( + LIB_NAME surround360kernels + CPP_SRCS + source/scanner_kernels/project_spherical_kernel_cpu.cpp + source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp + source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp + source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp + + PROTO_SRC source/scanner_kernels/surround360.proto) + target_link_libraries(surround360kernels PUBLIC + LibVrCamera + folly + ${OpenCV_LIBS}) +ENDIF() diff --git a/surround360_render/scripts/batch_process_video.py b/surround360_render/scripts/batch_process_video.py index 6be7741e..bfa927b4 100755 --- a/surround360_render/scripts/batch_process_video.py +++ b/surround360_render/scripts/batch_process_video.py @@ -82,6 +82,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l parser.add_argument('--rig_json_file', help='path to rig json file', required=True) parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True) parser.add_argument('--verbose', dest='verbose', action='store_true') + parser.add_argument('--cores', default=-1) parser.set_defaults(save_debug_images=False) parser.set_defaults(enable_top=False) parser.set_defaults(enable_bottom=False) @@ -192,6 +193,7 @@ def list_only_files(src_dir): return filter(lambda f: f[0] != ".", [f for f in l render_params["FINAL_EQR_HEIGHT"] = 6144 elif quality == "8k": render_params["SHARPENNING"] = 0.25 + render_params["SHARPENNING"] = 0.0 render_params["EQR_WIDTH"] = 8400 render_params["EQR_HEIGHT"] = 4096 render_params["FINAL_EQR_WIDTH"] = 8192 diff --git a/surround360_render/scripts/geometric_calibration.py b/surround360_render/scripts/geometric_calibration.py index ead2034d..cb907b50 100644 --- a/surround360_render/scripts/geometric_calibration.py +++ b/surround360_render/scripts/geometric_calibration.py @@ -36,6 +36,7 @@ {COLMAP_DIR}/feature_extractor --General.image_path {IMAGE_PATH} --General.database_path {COLMAP_DB_PATH} +--ExtractionOptions.gpu_index 0 """ COLMAP_MATCH_TEMPLATE = """ diff --git a/surround360_render/scripts/scanner_process_video.py b/surround360_render/scripts/scanner_process_video.py new file mode 100644 index 00000000..eab35b82 --- /dev/null +++ b/surround360_render/scripts/scanner_process_video.py @@ -0,0 +1,659 @@ +from scannerpy import Database, DeviceType, Job, BulkJob +from scannerpy.stdlib import NetDescriptor, parsers, bboxes +import numpy as np +import argparse +import os +import signal +import subprocess +import sys +import struct +import scipy.misc +import time + +from os import listdir, system +from os.path import isfile, join +from timeit import default_timer as timer + +current_process = None +def signal_term_handler(signal, frame): + if current_process: + print "Terminating process: " + current_process.name + "..." + current_process.terminate() + sys.exit(0) + +RENDER_COMMAND_TEMPLATE = """ +{SURROUND360_RENDER_DIR}/bin/TestRenderStereoPanorama +--logbuflevel -1 +--log_dir {LOG_DIR} +--stderrthreshold 0 +--v {VERBOSE_LEVEL} +--rig_json_file {RIG_JSON_FILE} +--imgs_dir {SRC_DIR}/rgb +--frame_number {FRAME_ID} +--output_data_dir {SRC_DIR} +--prev_frame_data_dir {PREV_FRAME_DIR} +--output_cubemap_path {OUT_CUBE_DIR}/cube_{FRAME_ID}.png +--output_equirect_path {OUT_EQR_DIR}/eqr_{FRAME_ID}.png +--cubemap_format {CUBEMAP_FORMAT} +--side_flow_alg {SIDE_FLOW_ALGORITHM} +--polar_flow_alg {POLAR_FLOW_ALGORITHM} +--poleremoval_flow_alg {POLEREMOVAL_FLOW_ALGORITHM} +--cubemap_width {CUBEMAP_WIDTH} +--cubemap_height {CUBEMAP_HEIGHT} +--eqr_width {EQR_WIDTH} +--eqr_height {EQR_HEIGHT} +--final_eqr_width {FINAL_EQR_WIDTH} +--final_eqr_height {FINAL_EQR_HEIGHT} +--interpupilary_dist 6.4 +--zero_parallax_dist 10000 +--sharpenning {SHARPENNING} +{EXTRA_FLAGS} +""" + +def start_subprocess(name, cmd): + global current_process + current_process = subprocess.Popen(cmd, shell=True) + current_process.name = name + current_process.communicate() + + +def project_tasks(db, videos, videos_idx): + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for i, (vid, vid_idx) in enumerate(zip(videos.tables(), + videos_idx.tables())): + task = db.protobufs.Task() + task.output_table_name = vid.name() + str(i) + + sample = task.samples.add() + sample.table_name = vid.name() + sample.column_names.extend([c.name() for c in vid.columns()]) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + tasks.append(task) + return tasks + + +def project_tasks_join(db, videos, videos_idx): + tasks = [] + sampler_args = db.protobufs.AllSamplerArgs() + sampler_args.warmup_size = 0 + sampler_args.sample_size = 32 + + for i in range(len(videos.tables())): + left_cam_idx = i + if i == len(videos.tables()) - 1: + right_cam_idx = 0 + else: + right_cam_idx = left_cam_idx + 1 + + vid_left = videos.tables(left_cam_idx) + vid_idx_left = videos_idx.tables(left_cam_idx) + vid_right = videos.tables(right_cam_idx) + vid_idx_right = videos_idx.tables(right_cam_idx) + + task = db.protobufs.Task() + task.output_table_name = vid_left.name() + str(i) + + sample = task.samples.add() + sample.table_name = vid_left.name() + sample.column_names.extend([c.name() for c in vid_left.columns()]) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx_left.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_right.name() + sample.column_names.extend(['frame']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + sample = task.samples.add() + sample.table_name = vid_idx_right.name() + sample.column_names.extend(['camera_index']) + sample.sampling_function = "All" + sample.sampling_args = sampler_args.SerializeToString() + + tasks.append(task) + return tasks + + +def project_images(db, videos, videos_idx, render_params): + jobs = [] + for i in range(len(video.tables())): + left_frame = videos.tables(i).as_op().all() + left_cam_idx = videos_idx.tables(i).as_op().all() + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + proj_frame = db.ops.ProjectSpherical(frame = frame, camra_id = cam_idx, + args=args) + + job = Job(columns = [proj_frame], + name = 'surround360_spherical_{:d}'.format(i)) + jobs.append(job) + + return db.run(jobs, 'surround360_spherical', force=True) + + +def compute_temporal_flow(db, overlap, render_params): + jobs = [] + for i in range(len(overlap.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(overlap.tables()) + + left_proj_frame = overlap.tables(left_cam_idx).as_op().all() + right_proj_frame = overlap.tables(right_cam_idx).as_op().all() + + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + args=args) + + job = Job(columns = [left_flow, right_flow], + name = 'surround360_flow_{:d}'.format(i)) + jobs.append(job) + + return db.run(jobs, 'surround360_flow', force=True) + + +def render_stereo_panorama_chunks(db, overlap, flow, render_params): + jobs = [] + for i in range(len(video.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(video.tables()) + + left_proj_frame = overlap.tables(left_cam_idx).as_op().all() + left_flow = flow.tables(left_cam_idx).as_op().all() + right_proj_frame = videos.tables(right_cam_idx).as_op().all() + right_flow = flow_idx.tables(right_cam_idx).as_op().all() + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + + job = Job(columns = [left_chunk, right_chunk], + name = 'surround360_chunk_{:d}'.format(i)) + jobs.append(job) + + return db.run(jobs, force=True) + + +def concat_stereo_panorama_chunks(db, chunks, render_params, is_left): + num_cams = 14 + item_size = 10 + print(chunks) + assert num_cams == len(chunks) + + left_inputs = [] + right_inputs = [] + for c in range(num_cams): + left_chunk = db.ops.FrameInput() + right_chunk = db.ops.FrameInput() + left_inputs.append(left_chunk) + right_inputs.append(right_chunk) + + args = db.protobufs.ConcatPanoramaChunksArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.final_eqr_width = render_params["FINAL_EQR_WIDTH"] + args.final_eqr_height = render_params["FINAL_EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = False + panorama = db.ops.ConcatPanoramaChunks( + *(left_inputs + right_inputs), + args=args) + output_op = db.ops.Output(columns=[panorama]) + + op_args = { + output_op: 'surround360_pano' + } + for c in range(num_cams): + op_args[left_inputs[c]] = chunks[c].column('left_chunk') + op_args[right_inputs[c]] = chunks[c].column('right_chunk') + + job = Job(op_args=op_args) + bulk_job = BulkJob(output=output_op, jobs=[job]) + + tables = db.run( + bulk_job, force=True, + work_packet_size=item_size, + io_packet_size=item_size, + pipeline_instances_per_node=render_params["CORES"]) + return tables[0] + +def fused_project_flow_chunk_concat(db, videos, videos_idx, render_params, + start, end): + jobs = [] + item_size = 10 + + proj_frames = [] + for i in range(len(videos.tables())): + left_cam_idx = i + frame = videos.tables(left_cam_idx).as_op().range(start, end, task_size=item_size) + cam_idx = videos_idx.tables(left_cam_idx).as_op().range(start, end, task_size=item_size) + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + proj_frame = db.ops.ProjectSpherical( + frame = frame, + camera_id = cam_idx, + args=args) + proj_frames.append(proj_frame) + + left_chunks = [] + right_chunks = [] + for i in range(len(videos.tables())): + left_cam_idx = i + right_cam_idx = (left_cam_idx + 1) % len(videos.tables()) + + left_proj_frame = proj_frames[left_cam_idx] + right_proj_frame = proj_frames[right_cam_idx] + + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + camera_rig_path = render_params["RIG_JSON_FILE"]) + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + left_chunks.append(left_chunk) + right_chunks.append(right_chunk) + + jobs = [] + args = db.protobufs.ConcatPanoramaChunksArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = True + panorama_left = db.ops.ConcatPanoramaChunks(*left_chunks, args=args) + + args = db.protobufs.ConcatPanoramaChunksArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + args.left = False + panorama_right = db.ops.ConcatPanoramaChunks(*right_chunks, args=args) + + job = Job(columns = [panorama_left, panorama_right], + name = 'surround360_pano_both') + return db.run(job, force=True) + + +def fused_project_flow_and_stereo_chunk(db, videos, videos_idx, render_params, start, end): + warmup_size = 10 + task_size = 100 + + left_frame = db.ops.FrameInput() + left_cam_idx = db.ops.Input() + + right_frame = db.ops.FrameInput() + right_cam_idx = db.ops.Input() + + args = db.protobufs.ProjectSphericalArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + left_proj_frame = db.ops.ProjectSpherical( + frame = left_frame, + camera_id = left_cam_idx, + args=args) + right_proj_frame = db.ops.ProjectSpherical( + frame = right_frame, + camera_id = right_cam_idx, + args=args) + + left_flow, right_flow = db.ops.TemporalOpticalFlow( + left_projected_frame = left_proj_frame, + right_projected_frame = right_proj_frame, + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + camera_rig_path = render_params["RIG_JSON_FILE"], + warmup = warmup_size) + + left_chunk, right_chunk = db.ops.RenderStereoPanoramaChunk( + left_projected_frame = left_proj_frame, + left_flow = left_flow, + right_projected_frame = right_proj_frame, + right_flow = right_flow, + eqr_width = render_params["EQR_WIDTH"], + eqr_height = render_params["EQR_HEIGHT"], + camera_rig_path = render_params["RIG_JSON_FILE"], + flow_algo = render_params["SIDE_FLOW_ALGORITHM"], + zero_parallax_dist = 10000, + interpupilary_dist = 6.4) + + left_chunk_sample = left_chunk.sample() + right_chunk_sample = right_chunk.sample() + + output_op = db.ops.Output(columns=[left_chunk_sample.lossless(), + right_chunk_sample.lossless()]) + + jobs = [] + for i in range(len(videos.tables())): + left_idx = i + right_idx = (left_idx + 1) % len(videos.tables()) + + sample = db.sampler.range(start, end) + + job = Job(op_args={ + left_frame: videos.tables(left_idx).column('frame'), + left_cam_idx: videos_idx.tables(left_idx).column('camera_index'), + right_frame: videos.tables(right_idx).column('frame'), + right_cam_idx: videos_idx.tables(right_idx).column('camera_index'), + left_chunk_sample: sample, + right_chunk_sample: sample, + output_op: 'surround360_chunk_{:d}'.format(i), + }) + jobs.append(job) + + bulk_job = BulkJob(output=output_op, jobs=jobs) + + return db.run(bulk_job, force=True, + work_packet_size=10, + io_packet_size=task_size, + pipeline_instances_per_node=render_params["CORES"]) + + +def fused_flow_and_stereo_chunk(db, overlap, render_params): + in_columns = ["index", + "projected_left", "frame_info_left", + "projected_right", "frame_info_right"] + input_op = db.ops.Input(in_columns) + + args = db.protobufs.TemporalOpticalFlowArgs() + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + temporal_op = db.ops.TemporalOpticalFlow(inputs=[(input_op, in_columns[1:])], + args=args) + + args = db.protobufs.RenderStereoPanoramaChunkArgs() + args.eqr_width = render_params["EQR_WIDTH"] + args.eqr_height = render_params["EQR_HEIGHT"] + args.camera_rig_path = render_params["RIG_JSON_FILE"] + args.flow_algo = render_params["SIDE_FLOW_ALGORITHM"] + args.zero_parallax_dist = 10000 + args.interpupilary_dist = 6.4 + op = db.ops.RenderStereoPanoramaChunk( + inputs=[(input_op, ["projected_left", "frame_info_left"]), + (temporal_op, ["flow_left", "frame_info_left"]), + (input_op, ["projected_right", "frame_info_right"]), + (temporal_op, ["flow_right", "frame_info_right"])], + args=args) + + tasks = flow_tasks(db, overlap) + + return db.run(tasks, op, 'surround360_chunk_fused', force=True) + + +if __name__ == "__main__": + signal.signal(signal.SIGTERM, signal_term_handler) + + parser = argparse.ArgumentParser(description='batch process video frames') + parser.add_argument('--root_dir', help='path to frame container dir', required=True) + parser.add_argument('--surround360_render_dir', help='project root path, containing bin and scripts dirs', required=False, default='.') + parser.add_argument('--start_frame', help='first frame index', required=True) + parser.add_argument('--end_frame', help='last frame index', required=True) + parser.add_argument('--quality', help='3k,4k,6k,8k', required=True) + parser.add_argument('--cubemap_width', help='default is to not generate cubemaps', required=False, default=0) + parser.add_argument('--cubemap_height', help='default is to not generate cubemaps', required=False, default=0) + parser.add_argument('--cubemap_format', help='photo,video', required=False, default='photo') + parser.add_argument('--save_debug_images', dest='save_debug_images', action='store_true') + parser.add_argument('--enable_top', dest='enable_top', action='store_true') + parser.add_argument('--enable_bottom', dest='enable_bottom', action='store_true') + parser.add_argument('--enable_pole_removal', dest='enable_pole_removal', action='store_true') + parser.add_argument('--resume', dest='resume', action='store_true', help='looks for a previous frame optical flow instead of starting fresh') + parser.add_argument('--rig_json_file', help='path to rig json file', required=True) + parser.add_argument('--flow_alg', help='flow algorithm e.g., pixflow_low, pixflow_search_20', required=True) + parser.add_argument('--verbose', dest='verbose', action='store_true') + parser.add_argument('--cores', default=-1, type=int) + parser.set_defaults(save_debug_images=False) + parser.set_defaults(enable_top=False) + parser.set_defaults(enable_bottom=False) + parser.set_defaults(enable_pole_removal=False) + args = vars(parser.parse_args()) + + root_dir = args["root_dir"] + surround360_render_dir = args["surround360_render_dir"] + log_dir = root_dir + "/logs" + out_eqr_frames_dir = root_dir + "/eqr_frames" + out_cube_frames_dir = root_dir + "/cube_frames" + flow_dir = root_dir + "/flow" + debug_dir = root_dir + "/debug" + min_frame = int(args["start_frame"]) + max_frame = int(args["end_frame"]) + cubemap_width = int(args["cubemap_width"]) + cubemap_height = int(args["cubemap_height"]) + cubemap_format = args["cubemap_format"] + quality = args["quality"] + save_debug_images = args["save_debug_images"] + enable_top = args["enable_top"] + enable_bottom = args["enable_bottom"] + enable_pole_removal = args["enable_pole_removal"] + resume = args["resume"] + rig_json_file = args["rig_json_file"] + flow_alg = args["flow_alg"] + verbose = args["verbose"] + + start_time = timer() + + frame_range = range(min_frame, max_frame + 1) + + render_params = { + "SURROUND360_RENDER_DIR": surround360_render_dir, + "LOG_DIR": log_dir, + "SRC_DIR": root_dir, + "PREV_FRAME_DIR": "NONE", + "OUT_EQR_DIR": out_eqr_frames_dir, + "OUT_CUBE_DIR": out_cube_frames_dir, + "CUBEMAP_WIDTH": cubemap_width, + "CUBEMAP_HEIGHT": cubemap_height, + "CUBEMAP_FORMAT": cubemap_format, + "RIG_JSON_FILE": rig_json_file, + "SIDE_FLOW_ALGORITHM": flow_alg, + "POLAR_FLOW_ALGORITHM": flow_alg, + "POLEREMOVAL_FLOW_ALGORITHM": flow_alg, + "EXTRA_FLAGS": "", + "CORES": args["cores"] + } + + + if save_debug_images: + render_params["EXTRA_FLAGS"] += " --save_debug_images" + + if enable_top: + render_params["EXTRA_FLAGS"] += " --enable_top" + + if enable_pole_removal and enable_bottom is False: + sys.stderr.write("Cannot use enable_pole_removal if enable_bottom is not used") + exit(1) + + if enable_bottom: + render_params["EXTRA_FLAGS"] += " --enable_bottom" + if enable_pole_removal: + render_params["EXTRA_FLAGS"] += " --enable_pole_removal" + render_params["EXTRA_FLAGS"] += " --bottom_pole_masks_dir " + root_dir + "/pole_masks" + + if quality == "3k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 3080 + render_params["EQR_HEIGHT"] = 1540 + render_params["FINAL_EQR_WIDTH"] = 3080 + render_params["FINAL_EQR_HEIGHT"] = 3080 + elif quality == "4k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 4200 + render_params["EQR_HEIGHT"] = 1024 + render_params["FINAL_EQR_WIDTH"] = 4096 + render_params["FINAL_EQR_HEIGHT"] = 2048 + elif quality == "6k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 6300 + render_params["EQR_HEIGHT"] = 3072 + render_params["FINAL_EQR_WIDTH"] = 6144 + render_params["FINAL_EQR_HEIGHT"] = 6144 + elif quality == "8k": + render_params["SHARPENNING"] = 0.25 + render_params["EQR_WIDTH"] = 8400 + render_params["EQR_HEIGHT"] = 4096 + render_params["FINAL_EQR_WIDTH"] = 8192 + render_params["FINAL_EQR_HEIGHT"] = 8192 + else: + sys.stderr.write("Unrecognized quality setting: " + quality) + exit(1) + + collection_name = 'surround360' + idx_collection_name = 'surround360_index' + db_start = timer() + with Database() as db: + print('DB', timer() - db_start) + db.load_op( + os.path.join(surround360_render_dir, + 'build/lib/libsurround360kernels.so'), + os.path.join(surround360_render_dir, + 'build/source/scanner_kernels/surround360_pb2.py')) + + if not db.has_collection(collection_name): + print "----------- [Render] loading surround360 collection" + sys.stdout.flush() + paths = [os.path.join(root_dir, 'rgb', 'cam{:d}'.format(i), 'vid.mp4') + for i in range(1, 15)] + collection, _ = db.ingest_video_collection(collection_name, paths, + force=True) + + idx_tables = [] + num_rows = collection.tables(0).num_rows() + columns = ['camera_index'] + for c in range(0, len(paths)): + rows = [] + for i in range(num_rows): + rows.append([struct.pack('i', c)]) + table_name = 'surround_cam_idx_{:d}'.format(c) + db.new_table(table_name, columns, rows, force=True) + idx_tables.append(db.table(table_name)) + db.new_collection(idx_collection_name, idx_tables, force=True) + + + videos = db.collection(collection_name) + videos_idx = db.collection(idx_collection_name) + + start_time = timer() + print "----------- [Render] processing frames ", min_frame, max_frame + sys.stdout.flush() + if verbose: + print(render_params) + sys.stdout.flush() + + visualize = True + fused_4 = False # Fused project, flow, stereo chunk, and pano + fused_3 = True # Fused project, flow, and stereo chunk + fused_2 = False # Fused flow and stereo chunk + fused_1 = False # Nothing fused + if fused_4: + pano_col = fused_project_flow_chunk_concat(db, videos, videos_idx, + render_params, + min_frame, max_frame + 1) + pano_col.profiler().write_trace('fused4.trace') + elif fused_3: + flow_stereo_start = timer() + chunk_col = fused_project_flow_and_stereo_chunk( + db, videos, videos_idx, render_params, min_frame, max_frame + 1) + print(db.summarize()) + chunk_col = [db.table('surround360_chunk_{:d}'.format(i)) + for i in range(14)] + print('Proj flow stereo', timer() - flow_stereo_start) + concat_start = timer() + pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, + True) + print('Concat', timer() - concat_start) + chunk_col[0].profiler().write_trace('fused3.trace') + pano_col.profiler().write_trace('fused3_concat.trace') + elif fused_2: + proj_start = timer() + proj_col = project_images(db, videos, videos_idx, render_params) + print('Proj', timer() - proj_start) + flow_stereo_start = timer() + chunk_col = fused_flow_and_stereo_chunk(db, proj_col, render_params) + print('Flow stereo', timer() - flow_stereo_start) + concat_start = timer() + pano_col = concat_stereo_panorama_chunks(db, chunk_col, render_params, + True) + print('Concat', timer() - concat_start) + elif fused_1: + flow_col = compute_temporal_flow(db, proj_col, render_params) + if save_debug_images: + t1 = flow_col.tables(0) + for fi, tup in t1.load(['flow_left', 'frame_info_left']): + frame_info = db.protobufs.FrameInfo() + frame_info.ParseFromString(tup[1]) + + frame = np.frombuffer(tup[0], dtype=np.float32).reshape( + frame_info.height, + frame_info.width, + 2) + frame = np.append(frame, np.zeros((frame_info.height, frame_info.width, 1)), axis=2) + scipy.misc.toimage(frame[:,:,:]).save('flow_test.png') + + chunk_col = render_stereo_panorama_chunks(db, proj_col, flow_col, render_params) + + png_start = timer() + pano_col = db.table('surround360_pano') + # left_table = pano_col[0] + # right_table = pano_col[1] + if visualize: + pano_col.column('panorama').save_mp4('pano_test', + scale=(2048, 2048)) + print('To png', timer() - png_start) + + end_time = timer() + + if verbose: + total_runtime = end_time - start_time + avg_runtime = total_runtime / float(max_frame - min_frame + 1) + print "Render total runtime:", total_runtime, "sec" + print "Average runtime:", avg_runtime, "sec/frame" + sys.stdout.flush() diff --git a/surround360_render/source/calibration/GeometricCalibration.cpp b/surround360_render/source/calibration/GeometricCalibration.cpp index a669c8bf..6c202090 100644 --- a/surround360_render/source/calibration/GeometricCalibration.cpp +++ b/surround360_render/source/calibration/GeometricCalibration.cpp @@ -9,6 +9,7 @@ #include "opencv2/stitching/detail/matchers.hpp" #include "ceres/ceres.h" #include "ceres/rotation.h" +#include "ceres/types.h" #include "GeometricCalibration.h" #include "Camera.h" @@ -40,6 +41,7 @@ DEFINE_double(perturb_principals, 0, "pertub principals (pixels)"); DEFINE_int64(experiments, 1, "calibrate multiple times"); DEFINE_bool(discard_outside_fov, true, "discard matches outside fov"); DEFINE_bool(save_debug_images, false, "save intermediate images"); +//#define FLAGS_save_debug_images false // is the stem expected to be the camera id? i.e. the path is of the form: // / ... /. @@ -767,6 +769,10 @@ void solve( std::vector& rotations, std::vector& traces) { ceres::Solver::Options options; + options.linear_solver_type = ceres::ITERATIVE_SCHUR; + options.preconditioner_type = ceres::SCHUR_JACOBI; + options.num_threads = 32; + options.num_linear_solver_threads = 32; options.use_inner_iterations = true; options.max_num_iterations = 500; options.minimizer_progress_to_stdout = false; @@ -804,13 +810,13 @@ void refine( triangulateTraces(traces, keypointMap, cameras); // visualization for debugging - showMatches( - cameras, - keypointMap, - overlaps, - traces, - debugDir, - pass); + // showMatches( + // cameras, + // keypointMap, + // overlaps, + // traces, + // debugDir, + // pass); // read camera parameters from cameras std::vector positions; diff --git a/surround360_render/source/camera_isp/Raw2Rgb.cpp b/surround360_render/source/camera_isp/Raw2Rgb.cpp index 594447b2..551591f3 100644 --- a/surround360_render/source/camera_isp/Raw2Rgb.cpp +++ b/surround360_render/source/camera_isp/Raw2Rgb.cpp @@ -403,6 +403,8 @@ int main(int argc, char* argv[]) { FLAGS_input_image_path, CV_LOAD_IMAGE_GRAYSCALE | CV_LOAD_IMAGE_ANYDEPTH); + LOG(INFO) << "Camera path " << FLAGS_input_image_path; + if (inputImage.cols > 2 && inputImage.rows > 2) { const uint8_t depth = inputImage.type() & CV_MAT_DEPTH_MASK; diff --git a/surround360_render/source/optical_flow/NovelView.h b/surround360_render/source/optical_flow/NovelView.h index eb6f97e5..0eb435e6 100644 --- a/surround360_render/source/optical_flow/NovelView.h +++ b/surround360_render/source/optical_flow/NovelView.h @@ -133,8 +133,17 @@ class NovelViewGenerator { const LazyNovelViewBuffer& lazyBuffer) = 0; // for debugging + virtual Mat getImageL() { return Mat(); } + virtual Mat getImageR() { return Mat(); } + + virtual void setImageL(const Mat& image) = 0; + virtual void setImageR(const Mat& image) = 0; + virtual Mat getFlowLtoR() { return Mat(); } virtual Mat getFlowRtoL() { return Mat(); } + + virtual void setFlowLtoR(const Mat& flow) = 0; + virtual void setFlowRtoL(const Mat& flow) = 0; }; // this is a base class for novel view generators that work by reduction to optical flow. @@ -162,8 +171,17 @@ class NovelViewGeneratorLazyFlow : public NovelViewGenerator { pair combineLazyNovelViews(const LazyNovelViewBuffer& lazyBuffer); + Mat getImageL() { return imageL; } + Mat getImageR() { return imageR; } + + void setImageL(const Mat& image) { imageL = image; } + void setImageR(const Mat& image) { imageR = image; } + Mat getFlowLtoR() { return flowLtoR; } Mat getFlowRtoL() { return flowRtoL; } + + void setFlowLtoR(const Mat& flow) { flowLtoR = flow; } + void setFlowRtoL(const Mat& flow) { flowRtoL = flow; } }; // the name "asymmetric" here refers to the idea that we compute an optical flow from diff --git a/surround360_render/source/render/RigDescription.cpp b/surround360_render/source/render/RigDescription.cpp index 8864906e..f68162c3 100644 --- a/surround360_render/source/render/RigDescription.cpp +++ b/surround360_render/source/render/RigDescription.cpp @@ -15,8 +15,11 @@ using namespace cv; using namespace std; using namespace surround360::util; -RigDescription::RigDescription(const string& filename) { - rig = Camera::loadRig(filename); +RigDescription::RigDescription(const string& filename) + : RigDescription(Camera::loadRig(filename)) { +} + +RigDescription::RigDescription(const Camera::Rig& rig) { for (const Camera& camera : rig) { if (camera.group.find("side") != string::npos) { rigSideOnly.emplace_back(camera); diff --git a/surround360_render/source/render/RigDescription.h b/surround360_render/source/render/RigDescription.h index a452e188..ca32a3ff 100644 --- a/surround360_render/source/render/RigDescription.h +++ b/surround360_render/source/render/RigDescription.h @@ -29,6 +29,8 @@ struct RigDescription { Camera::Rig rigSideOnly; RigDescription(const string& filename); + RigDescription(const Camera::Rig& rig); + // find the camera that is closest to pointing in the provided direction // ignore those with excessive distance from the camera axis to the rig center const Camera& findCameraByDirection( @@ -60,4 +62,29 @@ struct RigDescription { } }; +// measured in radians from forward +inline float approximateFov(const Camera& camera, const bool vertical) { + Camera::Vector2 a = camera.principal; + Camera::Vector2 b = camera.principal; + if (vertical) { + a.y() = 0; + b.y() = camera.resolution.y(); + } else { + a.x() = 0; + b.x() = camera.resolution.x(); + } + return acos(max( + camera.rig(a).direction().dot(camera.forward()), + camera.rig(b).direction().dot(camera.forward()))); +} + +// measured in radians from forward +inline float approximateFov(const Camera::Rig& rig, const bool vertical) { + float result = 0; + for (const auto& camera : rig) { + result = std::max(result, approximateFov(camera, vertical)); + } + return result; +} + } // namespace surround360 diff --git a/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp new file mode 100644 index 00000000..b0d24c08 --- /dev/null +++ b/surround360_render/source/scanner_kernels/concat_panorama_chunks_kernel_cpu.cpp @@ -0,0 +1,147 @@ +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { +namespace { +void padToheight(Mat& unpaddedImage, const int targetHeight) { + const int paddingAbove = (targetHeight - unpaddedImage.rows) / 2; + const int paddingBelow = targetHeight - unpaddedImage.rows - paddingAbove; + cv::copyMakeBorder( + unpaddedImage, + unpaddedImage, + paddingAbove, + paddingBelow, + 0, + 0, + BORDER_CONSTANT, + Scalar(0.0, 0.0, 0.0)); +} +} + +using namespace optical_flow; +using namespace math_util; + +class ConcatPanoramaChunksKernelCPU : public BatchedKernel, public VideoKernel { + public: + ConcatPanoramaChunksKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + rig_.reset(new RigDescription(args_.camera_rig_path())); + + num_chunks_ = config.input_columns.size() / 2; + } + + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + const int overlapImageWidth = + float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees); + + const float v = + atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f)); + const float psi = + asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius); + const float vergeAtInfinitySlabDisplacement = + psi * (float(camImageWidth) / fovHorizontalRadians); + const float theta = -M_PI / 2.0f + v + psi; + zeroParallaxNovelViewShiftPixels_ = + float(args_.eqr_width()) * (theta / (2.0f * M_PI)); + if (!args_.left()) { + zeroParallaxNovelViewShiftPixels_ *= -1; + } + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + check_frame(device_, input_columns[0][0]); + + i32 input_count = (i32)num_rows(input_columns[0]); + size_t output_image_width = (size_t)args_.final_eqr_width(); + size_t output_image_height = (size_t)args_.final_eqr_height(); + output_image_width += (output_image_width % 2); + output_image_height += (output_image_height % 2); + size_t output_image_size = + output_image_width * output_image_height * 3; + FrameInfo info(output_image_height, output_image_width, 3, FrameType::U8); + std::vector output_frames = new_frames(device_, info, input_count); + + std::vector left_pano_chunks(num_chunks_, Mat()); + std::vector right_pano_chunks(num_chunks_, Mat()); + cv::Mat left_pano; + cv::Mat right_pano; + for (i32 i = 0; i < input_count; ++i) { + for (i32 c = 0; c < num_chunks_; ++c) { + auto &left_chunk_col = input_columns[c]; + cv::cvtColor(frame_to_mat(left_chunk_col[i].as_const_frame()), + left_pano_chunks[c], CV_BGRA2BGR); + + auto &right_chunk_col = input_columns[num_chunks_ + c]; + cv::cvtColor(frame_to_mat(right_chunk_col[i].as_const_frame()), + right_pano_chunks[c], CV_BGRA2BGR); + } + left_pano = stackHorizontal(left_pano_chunks); + left_pano = + offsetHorizontalWrap(left_pano, zeroParallaxNovelViewShiftPixels_); + + right_pano = stackHorizontal(right_pano_chunks); + right_pano = + offsetHorizontalWrap(right_pano, -zeroParallaxNovelViewShiftPixels_); + + padToheight(left_pano, args_.eqr_height()); + padToheight(right_pano, args_.eqr_height()); + + resize(left_pano, left_pano, + Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0, + INTER_CUBIC); + resize(right_pano, right_pano, + Size(args_.final_eqr_width(), args_.final_eqr_height() / 2), 0, 0, + INTER_CUBIC); + + cv::Mat stereo_equirect = + stackVertical(vector({left_pano, right_pano})); + + u8 *output = output_frames[i]->data; + for (i32 r = 0; r < stereo_equirect.rows; ++r) { + memcpy(output + r * output_image_width * sizeof(char) * 3, + stereo_equirect.data + stereo_equirect.step * r, + stereo_equirect.cols * sizeof(char) * 3); + } + + insert_frame(output_columns[0], output_frames[i]); + } + } + + private: + surround360::proto::ConcatPanoramaChunksArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + int num_chunks_; + float zeroParallaxNovelViewShiftPixels_; +}; + +REGISTER_OP(ConcatPanoramaChunks) + .variadic_inputs() + .frame_output("panorama"); + +REGISTER_KERNEL(ConcatPanoramaChunks, ConcatPanoramaChunksKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp new file mode 100644 index 00000000..e7ea48ea --- /dev/null +++ b/surround360_render/source/scanner_kernels/project_spherical_kernel_cpu.cpp @@ -0,0 +1,102 @@ +#include "render/RigDescription.h" +#include "render/ImageWarper.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +#include + +using namespace scanner; + +namespace surround360 { + +class ProjectSphericalKernelCPU : public BatchedKernel, public VideoKernel { + public: + ProjectSphericalKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + // Initialize camera rig + rig_.reset(new RigDescription(args_.camera_rig_path())); + + hRadians_ = 2 * approximateFov(rig_->rigSideOnly, false); + vRadians_ = 2 * approximateFov(rig_->rigSideOnly, true); + } + + void reset() override { + is_reset_ = true; + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& frame_col = input_columns[0]; + auto& camera_id_col = input_columns[1]; + check_frame(device_, frame_col[0]); + + if (is_reset_) { + // Use the new camera id to update the spherical projection parameters + is_reset_ = false; + + camIdx_ = *((int*)camera_id_col[0].buffer); + const Camera& camera = rig_->rigSideOnly[camIdx_]; + + // the negative sign here is so the camera array goes clockwise + const int numCameras = 14; + float direction = -float(camIdx_) / float(numCameras) * 2.0f * M_PI; + leftAngle_ = direction + hRadians_ / 2; + rightAngle_ = direction - hRadians_ / 2; + topAngle_ = vRadians_ / 2; + bottomAngle_ = -vRadians_ / 2; + } + + i32 input_count = (i32)num_rows(frame_col); + size_t output_image_width = args_.eqr_width() * hRadians_ / (2 * M_PI); + size_t output_image_height = args_.eqr_height() * vRadians_ / M_PI; + size_t output_image_size = output_image_width * output_image_height * 4; + FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8); + std::vector output_frames = new_frames(device_, info, input_count); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat input = frame_to_mat(frame_col[i].as_const_frame()); + cv::Mat tmp; + cv::cvtColor(input, tmp, CV_BGR2BGRA); + + cv::Mat projection_image = frame_to_mat(output_frames[i]); + + surround360::warper::bicubicRemapToSpherical( + projection_image, tmp, rig_->rigSideOnly[camIdx_], leftAngle_, + rightAngle_, topAngle_, bottomAngle_); + + insert_frame(output_columns[0], output_frames[i]); + } + } + + private: + surround360::proto::ProjectSphericalArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + bool is_reset_ = true; + + float hRadians_; + float vRadians_; + + int camIdx_; + float leftAngle_; + float rightAngle_; + float topAngle_; + float bottomAngle_; +}; + +REGISTER_OP(ProjectSpherical) + .frame_input("frame") + .input("camera_id") + .frame_output("projected_frame"); + +REGISTER_KERNEL(ProjectSpherical, ProjectSphericalKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp new file mode 100644 index 00000000..c95b81ca --- /dev/null +++ b/surround360_render/source/scanner_kernels/render_stereo_panorama_chunk_kernel_cpu.cpp @@ -0,0 +1,156 @@ +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { + +using namespace optical_flow; +using namespace math_util; + +class RenderStereoPanoramaChunkKernelCPU : public BatchedKernel, + public VideoKernel { +public: + RenderStereoPanoramaChunkKernelCPU(const KernelConfig &config) + : BatchedKernel(config), device_(config.devices[0]) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + rig_.reset(new RigDescription(args_.camera_rig_path())); + + overlap_image_width_ = -1; + novel_view_gen_.reset( + new NovelViewGeneratorAsymmetricFlow(args_.flow_algo())); + } + + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + fov_horizontal_radians_ = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + overlap_image_width_ = float(camImageWidth) * + (overlapAngleDegrees / camFovHorizontalDegrees); + num_novel_views_ = + camImageWidth - overlap_image_width_; // per image pair + + const float v = + atanf(args_.zero_parallax_dist() / (args_.interpupilary_dist() / 2.0f)); + const float psi = + asinf(sinf(v) * (args_.interpupilary_dist() / 2.0f) / cameraRingRadius); + const float vergeAtInfinitySlabDisplacement = + psi * (float(camImageWidth) / fov_horizontal_radians_); + const float theta = -M_PI / 2.0f + v + psi; + const float zeroParallaxNovelViewShiftPixels = + float(args_.eqr_width()) * (theta / (2.0f * M_PI)); + + int currChunkX = 0; + lazy_view_buffer_.reset(new LazyNovelViewBuffer(args_.eqr_width() / numCams, + camImageHeight)); + for (int nvIdx = 0; nvIdx < num_novel_views_; ++nvIdx) { + const float shift = float(nvIdx) / float(num_novel_views_); + const float slabShift = + float(camImageWidth) * 0.5f - float(num_novel_views_ - nvIdx); + + for (int v = 0; v < camImageHeight; ++v) { + lazy_view_buffer_->warpL[currChunkX][v] = + Point3f(slabShift + vergeAtInfinitySlabDisplacement, v, shift); + lazy_view_buffer_->warpR[currChunkX][v] = + Point3f(slabShift - vergeAtInfinitySlabDisplacement, v, shift); + } + ++currChunkX; + } + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& left_frame_col = input_columns[0]; + auto& left_flow_col = input_columns[1]; + auto& right_frame_col = input_columns[2]; + auto& right_flow_col = input_columns[3]; + check_frame(device_, left_frame_col[0]); + assert(overlap_image_width_ != -1); + + i32 input_count = (i32)num_rows(left_frame_col); + size_t output_image_width = num_novel_views_; + size_t output_image_height = frame_info_.height(); + size_t output_image_size = + output_image_width * output_image_height * 4; + FrameInfo info(output_image_height, output_image_width, 4, FrameType::U8); + std::vector frames = new_frames(device_, info, input_count * 2); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame()); + cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame()); + cv::Mat left_overlap_input = + left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, + overlap_image_width_, left_input.rows)); + cv::Mat right_overlap_input = + right_input(cv::Rect(0, 0, + overlap_image_width_, right_input.rows)); + + + cv::Mat left_flow = frame_to_mat(left_flow_col[i].as_const_frame()); + cv::Mat right_flow = frame_to_mat(right_flow_col[i].as_const_frame()); + + // Initialize NovelViewGenerator with images and flow since we are + // bypassing the prepare method + novel_view_gen_->setImageL(left_overlap_input); + novel_view_gen_->setImageR(right_overlap_input); + novel_view_gen_->setFlowLtoR(left_flow); + novel_view_gen_->setFlowRtoL(right_flow); + std::pair lazyNovelChunksLR = + novel_view_gen_->combineLazyNovelViews(*lazy_view_buffer_.get()); + const cv::Mat& chunkL = lazyNovelChunksLR.first; + const cv::Mat& chunkR = lazyNovelChunksLR.second; + + u8* left_output = frames[2 * i]->data; + u8* right_output = frames[2 * i + 1]->data; + for (i32 r = 0; r < chunkL.rows; ++r) { + memcpy(left_output + r * output_image_width * sizeof(char) * 4, + chunkL.data + chunkL.step * r, + chunkL.cols * sizeof(char) * 4); + memcpy(right_output + r * output_image_width * sizeof(char) * 4, + chunkR.data + chunkR.step * r, + chunkR.cols * sizeof(char) * 4); + } + + insert_frame(output_columns[0], frames[2 * i]); + insert_frame(output_columns[1], frames[2 * i + 1]); + } + } + + private: + surround360::proto::RenderStereoPanoramaChunkArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + float fov_horizontal_radians_; + int overlap_image_width_; + int num_novel_views_; + + std::unique_ptr novel_view_gen_; + std::unique_ptr lazy_view_buffer_; +}; + +REGISTER_OP(RenderStereoPanoramaChunk) + .frame_input("left_projected_frame") + .frame_input("left_flow") + .frame_input("right_projected_frame") + .frame_input("right_flow") + .frame_output("left_chunk") + .frame_output("right_chunk"); + +REGISTER_KERNEL(RenderStereoPanoramaChunk, RenderStereoPanoramaChunkKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/scanner_kernels/surround360.proto b/surround360_render/source/scanner_kernels/surround360.proto new file mode 100644 index 00000000..1afc20fb --- /dev/null +++ b/surround360_render/source/scanner_kernels/surround360.proto @@ -0,0 +1,34 @@ +syntax = "proto3"; + +package surround360.proto; + +message ProjectSphericalArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + string camera_rig_path = 3; +} + +message TemporalOpticalFlowArgs { + string flow_algo = 1; + string camera_rig_path = 2; +} + +message RenderStereoPanoramaChunkArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + string camera_rig_path = 3; + string flow_algo = 4; + float zero_parallax_dist = 5; + float interpupilary_dist = 6; +} + +message ConcatPanoramaChunksArgs { + int32 eqr_height = 1; + int32 eqr_width = 2; + int32 final_eqr_height = 7; + int32 final_eqr_width = 8; + string camera_rig_path = 3; + float zero_parallax_dist = 4; + float interpupilary_dist = 5; + bool left = 6; +} diff --git a/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp new file mode 100644 index 00000000..160d3cd3 --- /dev/null +++ b/surround360_render/source/scanner_kernels/temporal_optical_flow_kernel_cpu.cpp @@ -0,0 +1,129 @@ +#include "render/RigDescription.h" +#include "optical_flow/NovelView.h" +#include "util/MathUtil.h" +#include "source/scanner_kernels/surround360.pb.h" + +#include "scanner/api/kernel.h" +#include "scanner/api/op.h" +#include "scanner/util/memory.h" +#include "scanner/util/opencv.h" + +using namespace scanner; + +namespace surround360 { + +using namespace optical_flow; +using namespace math_util; + +class TemporalOpticalFlowKernelCPU : public BatchedKernel, public VideoKernel { + public: + TemporalOpticalFlowKernelCPU(const KernelConfig& config) + : BatchedKernel(config), + device_(config.devices[0]) { + args_.ParseFromArray(config.args.data(), config.args.size()); + + rig_.reset(new RigDescription(args_.camera_rig_path())); + + overlap_image_width_ = -1; + novel_view_gen_.reset( + new NovelViewGeneratorAsymmetricFlow(args_.flow_algo())); + } + + void reset() override { + prev_frame_flow_l_to_r_ = cv::Mat(); + prev_frame_flow_r_to_l_ = cv::Mat(); + prev_overlap_image_l_ = cv::Mat(); + prev_overlap_image_r_ = cv::Mat(); + } + + void new_frame_info() override { + const int numCams = 14; + const float cameraRingRadius = rig_->getRingRadius(); + const float camFovHorizontalDegrees = + 2 * approximateFov(rig_->rigSideOnly, false) * (180 / M_PI); + const float fovHorizontalRadians = toRadians(camFovHorizontalDegrees); + const float overlapAngleDegrees = + (camFovHorizontalDegrees * float(numCams) - 360.0) / float(numCams); + const int camImageWidth = frame_info_.width(); + const int camImageHeight = frame_info_.height(); + overlap_image_width_ = + float(camImageWidth) * (overlapAngleDegrees / camFovHorizontalDegrees); + } + + void execute(const BatchedColumns& input_columns, + BatchedColumns& output_columns) override { + auto& left_frame_col = input_columns[0]; + auto& right_frame_col = input_columns[1]; + check_frame(device_, left_frame_col[0]); + assert(overlap_image_width_ != -1); + + i32 input_count = (i32)num_rows(left_frame_col); + size_t output_image_width = overlap_image_width_; + size_t output_image_height = frame_info_.height(); + size_t output_image_size = + output_image_width * output_image_height * 2 * sizeof(float); + FrameInfo info(output_image_height, output_image_width, 2, FrameType::F32); + std::vector frames = new_frames(device_, info, input_count * 2); + + for (i32 i = 0; i < input_count; ++i) { + cv::Mat left_input = frame_to_mat(left_frame_col[i].as_const_frame()); + cv::Mat right_input = frame_to_mat(right_frame_col[i].as_const_frame()); + + cv::Mat left_overlap_input = + left_input(cv::Rect(left_input.cols - overlap_image_width_, 0, + overlap_image_width_, left_input.rows)); + cv::Mat right_overlap_input = + right_input(cv::Rect(0, 0, + overlap_image_width_, right_input.rows)); + + novel_view_gen_->prepare(left_overlap_input, right_overlap_input, + prev_frame_flow_l_to_r_, prev_frame_flow_r_to_l_, + prev_overlap_image_l_, prev_overlap_image_r_); + + left_overlap_input.copyTo(prev_overlap_image_l_); + right_overlap_input.copyTo(prev_overlap_image_r_); + prev_frame_flow_l_to_r_ = novel_view_gen_->getFlowLtoR(); + prev_frame_flow_r_to_l_ = novel_view_gen_->getFlowRtoL(); + + u8* left_output = frames[2 * i]->data; + u8* right_output = frames[2 * i + 1]->data; + const auto& left_flow = novel_view_gen_->getFlowLtoR(); + const auto& right_flow = novel_view_gen_->getFlowRtoL(); + for (i32 r = 0; r < left_flow.rows; ++r) { + memcpy(left_output + r * output_image_width * sizeof(float) * 2, + left_flow.data + left_flow.step * r, + left_flow.cols * sizeof(float) * 2); + memcpy(right_output + r * output_image_width * sizeof(float) * 2, + right_flow.data + right_flow.step * r, + right_flow.cols * sizeof(float) * 2); + } + + insert_frame(output_columns[0], frames[2 * i]); + insert_frame(output_columns[1], frames[2 * i + 1]); + } + } + + private: + surround360::proto::TemporalOpticalFlowArgs args_; + std::unique_ptr rig_; + DeviceHandle device_; + int overlap_image_width_; + + std::unique_ptr novel_view_gen_; + cv::Mat prev_frame_flow_l_to_r_; + cv::Mat prev_frame_flow_r_to_l_; + cv::Mat prev_overlap_image_l_; + cv::Mat prev_overlap_image_r_; +}; + +REGISTER_OP(TemporalOpticalFlow) + .frame_input("left_projected_frame") + .frame_input("right_projected_frame") + .frame_output("left_flow") + .frame_output("right_flow") + .bounded_state(); + +REGISTER_KERNEL(TemporalOpticalFlow, TemporalOpticalFlowKernelCPU) + .device(DeviceType::CPU) + .num_devices(1); +} diff --git a/surround360_render/source/test/TestRenderStereoPanorama.cpp b/surround360_render/source/test/TestRenderStereoPanorama.cpp index 0f302feb..f6b5f669 100644 --- a/surround360_render/source/test/TestRenderStereoPanorama.cpp +++ b/surround360_render/source/test/TestRenderStereoPanorama.cpp @@ -71,31 +71,6 @@ DEFINE_string(cubemap_format, "video", "either video or photo const Camera::Vector3 kGlobalUp = Camera::Vector3::UnitZ(); -// measured in radians from forward -float approximateFov(const Camera& camera, const bool vertical) { - Camera::Vector2 a = camera.principal; - Camera::Vector2 b = camera.principal; - if (vertical) { - a.y() = 0; - b.y() = camera.resolution.y(); - } else { - a.x() = 0; - b.x() = camera.resolution.x(); - } - return acos(max( - camera.rig(a).direction().dot(camera.forward()), - camera.rig(b).direction().dot(camera.forward()))); -} - -// measured in radians from forward -float approximateFov(const Camera::Rig& rig, const bool vertical) { - float result = 0; - for (const auto& camera : rig) { - result = std::max(result, approximateFov(camera, vertical)); - } - return result; -} - void projectSideToSpherical( Mat& dst, const Mat& src,