Add Celebrity Recognition Tab

components/celebrity_recognition.js

@@ -0,0 +1,285 @@
+console.log('IMPORTING CELEBRITY RECOGNITION')
+
+// define style rules to be programtically loaded
+var style = document.createElement('style');
+style.innerHTML = `
+
+
+`;
+document.getElementsByTagName('head')[0].appendChild(style);
+
+
+
+// define component
+Vue.component('celebrity-recognition-viz', {
+    props: ['json_data', 'video_info'],
+    data: function () {
+        return {
+            confidence_threshold: 0.5,
+            interval_timer: null,
+            ctx: null
+        }
+    },
+    computed: {
+
+        celebrity_tracks: function () {
+            `
+            Extract just the logo tracking data from json
+            `
+
+            if (!this.json_data.annotation_results) return []
+
+            const tracksWithCelebrities = []
+            for (let index = 0; index < this.json_data.annotation_results.length; index++) {
+                if ('celebrity_recognition_annotations' in this.json_data.annotation_results[index] && 'celebrity_tracks' in this.json_data.annotation_results[index].celebrity_recognition_annotations) {
+                    const celebrityTracks = this.json_data.annotation_results[index].celebrity_recognition_annotations.celebrity_tracks
+                    celebrityTracks.forEach(track => {
+                        if ('celebrities' in track) {
+                            track.celebrities.forEach(() => {
+                                tracksWithCelebrities.push(track)
+                            })
+                        }
+                    })
+                }
+            }
+            return tracksWithCelebrities
+        },
+
+        indexed_celebrity_tracks: function () {
+            `
+            Create a clean list of celebrity tracking data with realisied nullable fields 
+            and scaled bounding boxes ready to be drawn by the canvas
+            `
+
+            const indexed_tracks = []
+
+            this.celebrity_tracks.forEach(element => {
+                const detected_celebrity = new Celebrity_Detected(element, this.video_info.height, this.video_info.width, this.confidence_threshold)
+                if (detected_celebrity.segments.length > 0)
+                    indexed_tracks.push(detected_celebrity)
+            })
+
+            return indexed_tracks
+        },
+
+        object_track_segments: function () {
+            ` 
+            create the list of cronological time segments that represent just when objects are present on screen
+            `
+            const segments = {}
+
+            this.indexed_celebrity_tracks.forEach(object_tracks => {
+
+                if (!(object_tracks.name in segments))
+                    segments[object_tracks.name] = { 'segments': [], 'count': 0 }
+
+
+                object_tracks.segments.forEach(celebrity_segemnt => {
+
+
+                    segments[object_tracks.name].count++
+
+                    var added = false
+
+                    for (let index = 0; index < segments[object_tracks.name].length; index++) {
+
+                        const segment = segments[object_tracks.name].segments[index]
+                        if (celebrity_segemnt.start_time < segment[1]) {
+                            segments[object_tracks.name].segments[index][1] = Math.max(segments[object_tracks.name].segments[index][1], celebrity_segemnt.end_time)
+                            added = true
+                            break
+                        }
+                    }
+
+                    if (!added)
+                        segments[object_tracks.name].segments.push([celebrity_segemnt.start_time, celebrity_segemnt.end_time])
+
+
+
+                })
+
+            })
+
+            return segments
+        }
+    },
+    methods: {
+        segment_style: function (segment) {
+            return {
+                left: ((segment[0] / this.video_info.length) * 100).toString() + '%',
+                width: (((segment[1] - segment[0]) / this.video_info.length) * 100).toString() + '%'
+            }
+        },
+        segment_clicked: function (segment_data) {
+            this.$emit('segment-clicked', { seconds: segment_data[0] - 0.5 })
+        }
+    },
+    template: `
+    <div calss="object-tracking-container">
+
+        <div class="confidence">
+            <span>Confidence threshold</span>
+            <input type="range" min="0.0" max="1" value="0.5" step="0.01" v-model="confidence_threshold">
+            <span class="confidence-value">{{confidence_threshold}}</span>
+        </div>
+
+        <div class="data-warning" v-if="celebrity_tracks.length == 0"> No celebrity detection data in JSON</div>
+
+        <transition-group name="segments" tag="div">
+
+            <div class="segment-container" v-for="segments, key in object_track_segments" v-bind:key="key + 'z'">
+                <div class="label">{{key}} ({{segments.count}})</div>
+                <div class="segment-timeline">
+                    <div class="segment" v-for="segment in segments.segments" 
+                                        v-bind:style="segment_style(segment)" 
+                                        v-on:click="segment_clicked(segment)"
+                    ></div>
+                </div>
+            </div>
+        </transition-group>
+    </div>
+    `,
+    mounted: function () {
+        console.log('mounted component')
+        var canvas = document.getElementById("my_canvas")
+        this.ctx = canvas.getContext("2d")
+        this.ctx.font = "20px Roboto"
+        const ctx = this.ctx
+
+        const component = this
+
+        this.interval_timer = setInterval(function () {
+            console.log('running')
+            const object_tracks = component.indexed_celebrity_tracks
+
+            draw_bounding_boxes(object_tracks, ctx)
+        }, 1000 / 30)
+    },
+    beforeDestroy: function () {
+        console.log('destroying component')
+        clearInterval(this.interval_timer)
+        this.ctx.clearRect(0, 0, 800, 500)
+    }
+})
+
+
+
+class Celebrity_Frame {
+
+    constructor(json_data, video_height, video_width) {
+
+        this.time_offset = nullable_time_offset_to_seconds(json_data.time_offset)
+
+        this.box = {
+            'x': (json_data.normalized_bounding_box.left || 0) * video_width,
+            'y': (json_data.normalized_bounding_box.top || 0) * video_height,
+            'width': ((json_data.normalized_bounding_box.right || 0) - (json_data.normalized_bounding_box.left || 0)) * video_width,
+            'height': ((json_data.normalized_bounding_box.bottom || 0) - (json_data.normalized_bounding_box.top || 0)) * video_height
+        }
+    }
+}
+
+
+
+class Celebrity_Track {
+    constructor(json_data, video_height, video_width) {
+
+        this.start_time = nullable_time_offset_to_seconds(json_data.face_track.segment.start_time_offset)
+        this.end_time = nullable_time_offset_to_seconds(json_data.face_track.segment.end_time_offset)
+        this.confidence = json_data.confidence
+
+        this.frames = []
+
+        json_data.face_track.timestamped_objects.forEach(frame => {
+            this.frames.push(new Celebrity_Frame(frame, video_height, video_width))
+        })
+    }
+
+    has_frames_for_time(seconds) {
+        return ((this.start_time <= seconds) && (this.end_time >= seconds))
+    }
+
+    most_recent_real_bounding_box(seconds) {
+
+        for (let index = 0; index < this.frames.length; index++) {
+            if (this.frames[index].time_offset > seconds) {
+                if (index > 0)
+                    return this.frames[index - 1].box
+                else
+                    return null
+            }
+        }
+        return null
+    }
+
+    most_recent_interpolated_bounding_box(seconds) {
+
+        for (let index = 0; index < this.frames.length; index++) {
+            if (this.frames[index].time_offset > seconds) {
+                if (index > 0) {
+                    if ((index == 1) || (index == this.frames.length - 1))
+                        return this.frames[index - 1].box
+
+                    // create a new interpolated box between the 
+                    const start_box = this.frames[index - 1]
+                    const end_box = this.frames[index]
+                    const time_delt_ratio = (seconds - start_box.time_offset) / (end_box.time_offset - start_box.time_offset)
+
+                    const interpolated_box = {
+                        'x': start_box.box.x + (end_box.box.x - start_box.box.x) * time_delt_ratio,
+                        'y': start_box.box.y + (end_box.box.y - start_box.box.y) * time_delt_ratio,
+                        'width': start_box.box.width + (end_box.box.width - start_box.box.width) * time_delt_ratio,
+                        'height': start_box.box.height + (end_box.box.height - start_box.box.height) * time_delt_ratio
+                    }
+                    return interpolated_box
+
+                } else
+                    return null
+            }
+        }
+        return null
+    }
+
+    current_bounding_box(seconds, interpolate = true) {
+
+        if (interpolate)
+            return this.most_recent_interpolated_bounding_box(seconds)
+        else
+            return this.most_recent_real_bounding_box(seconds)
+    }
+}
+
+
+class Celebrity_Detected {
+    constructor(track, video_height, video_width, confidence_threshold) {
+
+        this.segments = []
+
+        track.celebrities.forEach(celebrity => {
+            // Some outputs do not contain a confidence
+            if (!celebrity.confidence || celebrity.confidence > confidence_threshold) {
+                this.name = celebrity.celebrity.display_name
+                this.id = celebrity.celebrity.name
+                this.segments.push(new Celebrity_Track(track, video_height, video_width))
+            }
+        })
+    }
+
+    has_frames_for_time(seconds) {
+
+        for (let index = 0; index < this.segments.length; index++) {
+            if (this.segments[index].has_frames_for_time(seconds))
+                return true
+        }
+        return false
+    }
+
+    current_bounding_box(seconds, interpolate = true) {
+
+        for (let index = 0; index < this.segments.length; index++) {
+            if (this.segments[index].has_frames_for_time(seconds))
+                return this.segments[index].current_bounding_box(seconds, interpolate)
+        }
+        return null
+    }
+}

index.html

@@ -248,6 +248,11 @@ <h5>Try with your data</h5>
             v-on:shot-clicked="jump_video">
         </explicit-content-detection-viz>
 
+        <celebrity-recognition-viz v-if="current_view == 'Celebrity Recognition'"
+            id="celebrity_recognition" v-bind:json_data="json_data" v-bind:video_info="video_info"
+            v-on:segment-clicked="jump_video">
+        </celebrity-recognition-viz>
+
     </div>
 
 
@@ -264,6 +269,7 @@ <h5>Try with your data</h5>
     <script src="components/logo_recognition.js"></script>
     <script src="components/text_detection.js"></script>
     <script src="components/explicit_content_detection.js"></script>
+    <script src="components/celebrity_recognition.js"></script>
 
     <script>
 
@@ -334,7 +340,8 @@ <h5>Try with your data</h5>
                     'Logo Recognition': 'logo_recognition_annotations',
                     'Speech Transcription': 'speech_transcriptions',
                     'Text Detection': 'text_annotations',
-                    'Explicit Content Detection': 'explicit_annotation'
+                    'Explicit Content Detection': 'explicit_annotation',
+                    'Celebrity Recognition': 'celebrity_recognition_annotations'
                 }
             }, methods: {
                 jump_video: function (event_data) {

run_video_intelligence.py

@@ -18,7 +18,8 @@
     videointelligence.Feature.EXPLICIT_CONTENT_DETECTION,
     videointelligence.Feature.TEXT_DETECTION,
     videointelligence.Feature.FACE_DETECTION,
-    videointelligence.Feature.PERSON_DETECTION
+    videointelligence.Feature.PERSON_DETECTION,
+    videointelligence.Feature.CELEBRITY_RECOGNITION
 ]
 
 transcript_config = videointelligence.SpeechTranscriptionConfig(