diff --git a/README.md b/README.md index a6120bb2..6788a1f0 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ You can use datasets easily because you can access each dataset with multiple wa * House of Councillors of Japan * House of Representatives of Japan * Iris Dataset +* JMRD (Japanese Movie Recommendation Dialogue Dataset) * Libsvm * MNIST database * Mushroom diff --git a/example/jmrd.rb b/example/jmrd.rb new file mode 100755 index 00000000..7ed94321 --- /dev/null +++ b/example/jmrd.rb @@ -0,0 +1,36 @@ +#!/usr/bin/env ruby + +require 'datasets' + +jmrd = Datasets::JMRD.new(type: :train) + +jmrd.each do |dialogue| + puts "=" * 80 + puts "Dialogue ID: #{dialogue.dialog_id}" + puts "Movie: #{dialogue.movie_title}" + puts "First Speaker: #{dialogue.first_speaker}" + puts + + if dialogue.knowledge + puts "Knowledge:" + puts " Title: #{dialogue.knowledge.title}" + puts " Year: #{dialogue.knowledge.year}" + puts " Director: #{dialogue.knowledge.director_name}" + puts " Genres: #{dialogue.knowledge.genres.join(', ')}" if dialogue.knowledge.genres + puts + end + + puts "Dialogue:" + dialogue.utterances.each do |utterance| + speaker_label = utterance.speaker == "recommender" ? "[R]" : "[S]" + puts " #{speaker_label} #{utterance.text}" + + if utterance.checked_knowledge&.any? + knowledge_types = utterance.checked_knowledge.map { |ck| ck.type }.join(", ") + puts " (knowledge: #{knowledge_types})" + end + end + + # Show only first dialogue as example + break +end diff --git a/lib/datasets/jmrd.rb b/lib/datasets/jmrd.rb new file mode 100644 index 00000000..bc9a5314 --- /dev/null +++ b/lib/datasets/jmrd.rb @@ -0,0 +1,188 @@ +require "json" + +require_relative "dataset" + +module Datasets + class JMRD < Dataset + Dialogue = Struct.new( + :dialog_id, + :movie_title, + :first_speaker, + :questionnaire, + :knowledge, + :utterances + ) + + Questionnaire = Struct.new( + :recommender, + :seeker + ) + + QuestionnaireAnswers = Struct.new( + :q1, + :q2, + :q3, + :q4, + :q5 + ) + + Knowledge = Struct.new( + :no_knowledge, + :title, + :year, + :director_name, + :director_description, + :cast_names, + :cast_descriptions, + :genres, + :reviews, + :synopsis + ) + + Utterance = Struct.new( + :utterance_id, + :speaker, + :text, + :checked_knowledge + ) + + CheckedKnowledge = Struct.new( + :type, + :content + ) + + def initialize(type: :train) + unless [:train, :valid, :test].include?(type) + raise ArgumentError, ":type must be one of [:train, :valid, :test]: #{type.inspect}" + end + + super() + @metadata.id = "jmrd" + @metadata.name = "Japanese Movie Recommendation Dialogue Dataset (JMRD)" + @metadata.url = "https://github.com/ku-nlp/JMRD" + @metadata.licenses = ["CC-BY-SA-4.0"] + @metadata.description = <<~DESCRIPTION + JMRD (Japanese Movie Recommendation Dialogue Dataset) is a Japanese + knowledge-grounded dialogue dataset consisting of annotated movie + recommendation dialogues between humans. Every recommender's utterance + is associated with movie information as external knowledge. + + The dataset consists of about 5,000 dialogues between crowdworkers, + each of which consists of 23 utterances on average. All dialogues in + this dataset are divided into the train (4,575 dialogues), valid + (200 dialogues), and test sets (300 dialogues). + + Published in the 2nd DialDoc Workshop on Document-grounded Dialogue + and Conversational Question Answering, 2022. + + Reference: + Takashi Kodama, Ribeka Tanaka, and Sadao Kurohashi. + "Construction of Hierarchical Structured Knowledge-based Recommendation + Dialogue Dataset and Dialogue System." + DESCRIPTION + + @type = type + end + + def each + return to_enum(__method__) unless block_given? + + open_data do |json_data| + json_data.each do |dialogue_data| + yield build_dialogue(dialogue_data) + end + end + end + + private + + def open_data + data_path = cache_dir_path + "#{@type}.json" + data_url = "https://raw.githubusercontent.com/ku-nlp/JMRD/main/data/#{@type}.json" + download(data_path, data_url) + + json_data = JSON.parse(File.read(data_path)) + yield json_data + end + + def build_dialogue(data) + Dialogue.new( + data["dialog_id"], + data["movie_title"], + data["first_speaker"], + build_questionnaire(data["questionnaire"]), + build_knowledge(data["knowledge"]), + build_utterances(data["dialog"]) + ) + end + + def build_questionnaire(data) + return if data.nil? + + Questionnaire.new( + build_questionnaire_answers(data["recommender"]), + build_questionnaire_answers(data["seeker"]) + ) + end + + def build_questionnaire_answers(data) + return nil if data.nil? + + QuestionnaireAnswers.new( + data["Q1"], + data["Q2"], + data["Q3"], + data["Q4"], + data["Q5"] + ) + end + + def build_knowledge(data) + return nil if data.nil? + + Knowledge.new( + data["[知識なし]"], + data["タイトル"], + data["製作年度"], + data["監督名"], + data["監督説明"], + data["キャスト名"], + data["キャスト説明"], + data["ジャンル"], + data["レビュー"], + data["あらすじ"] + ) + end + + def build_utterances(data) + return [] if data.nil? + + data.map do |utterance_data| + build_utterance(utterance_data) + end + end + + def build_checked_knowledge(data) + CheckedKnowledge.new( + data["type"], + data["content"] + ) + end + + def build_utterance(data) + checked_knowledge = nil + if data["checked_knowledge"] + checked_knowledge = data["checked_knowledge"].map do |ck| + build_checked_knowledge(ck) + end + end + + Utterance.new( + data["utterance_id"], + data["speaker"], + data["text"], + checked_knowledge + ) + end + end +end diff --git a/lib/datasets/lazy.rb b/lib/datasets/lazy.rb index 204a240b..27351b96 100644 --- a/lib/datasets/lazy.rb +++ b/lib/datasets/lazy.rb @@ -61,6 +61,7 @@ def const_missing(name) LAZY_LOADER.register(:HouseOfRepresentative, "datasets/house-of-representative") LAZY_LOADER.register(:Iris, "datasets/iris") LAZY_LOADER.register(:ITACorpus, "datasets/ita-corpus") + LAZY_LOADER.register(:JMRD, "datasets/jmrd") LAZY_LOADER.register(:KuzushijiMNIST, "datasets/kuzushiji-mnist") LAZY_LOADER.register(:LIBSVM, "datasets/libsvm") LAZY_LOADER.register(:LIBSVMDatasetList, "datasets/libsvm-dataset-list") diff --git a/sig/datasets/jmrd.rbs b/sig/datasets/jmrd.rbs new file mode 100644 index 00000000..96fb1b3e --- /dev/null +++ b/sig/datasets/jmrd.rbs @@ -0,0 +1,120 @@ +module Datasets + class JMRD < Dataset + class CheckedKnowledge < Struct[untyped] + attr_accessor type(): String? + attr_accessor content(): String? + + def initialize: (?String? type, ?String? content) -> void + end + + class Utterance < Struct[untyped] + attr_accessor utterance_id(): String? + attr_accessor speaker(): String? + attr_accessor text(): String? + attr_accessor checked_knowledge(): Array[CheckedKnowledge]? + + def initialize: ( + ?String? utterance_id, + ?String? speaker, + ?String? text, + ?Array[CheckedKnowledge]? checked_knowledge + ) -> void + end + + class Knowledge < Struct[untyped] + attr_accessor no_knowledge(): String? + attr_accessor title(): String? + attr_accessor year(): String? + attr_accessor director_name(): String? + attr_accessor director_description(): String? + attr_accessor cast_names(): Array[String]? + attr_accessor cast_descriptions(): Array[String]? + attr_accessor genres(): Array[String]? + attr_accessor reviews(): Array[String]? + attr_accessor synopsis(): Array[String]? + + def initialize: ( + ?String? no_knowledge, + ?String? title, + ?String? year, + ?String? director_name, + ?String? director_description, + ?Array[String]? cast_names, + ?Array[String]? cast_descriptions, + ?Array[String]? genres, + ?Array[String]? reviews, + ?Array[String]? synopsis + ) -> void + end + + class QuestionnaireAnswers < Struct[untyped] + attr_accessor q1(): Integer? + attr_accessor q2(): Integer? + attr_accessor q3(): Integer? + attr_accessor q4(): Integer? + attr_accessor q5(): Integer? + + def initialize: ( + ?Integer? q1, + ?Integer? q2, + ?Integer? q3, + ?Integer? q4, + ?Integer? q5 + ) -> void + end + + class Questionnaire < Struct[untyped] + attr_accessor recommender(): QuestionnaireAnswers? + attr_accessor seeker(): QuestionnaireAnswers? + + def initialize: ( + ?QuestionnaireAnswers? recommender, + ?QuestionnaireAnswers? seeker + ) -> void + end + + class Dialogue < Struct[untyped] + attr_accessor dialog_id(): String? + attr_accessor movie_title(): String? + attr_accessor first_speaker(): String? + attr_accessor questionnaire(): Questionnaire? + attr_accessor knowledge(): Knowledge? + attr_accessor utterances(): Array[Utterance] + + def initialize: ( + ?String? dialog_id, + ?String? movie_title, + ?String? first_speaker, + ?Questionnaire? questionnaire, + ?Knowledge? knowledge, + Array[Utterance] utterances + ) -> void + end + + @type: Symbol + @metadata: Metadata + + def initialize: (?type: Symbol) -> void + + def each: () { (Dialogue) -> void } -> void + | () -> Enumerator[Dialogue, void] + + private + + def open_data: () { (Array[Hash[String, untyped]]) -> void } -> void + + def build_dialogue: (Hash[String, untyped] data) -> Dialogue + + def build_questionnaire: (Hash[String, untyped]? data) -> Questionnaire? + + def build_questionnaire_answers: (Hash[String, untyped]? data) -> QuestionnaireAnswers? + + def build_knowledge: (Hash[String, untyped]? data) -> Knowledge? + + def build_utterances: (Array[Hash[String, untyped]]? data) -> Array[Utterance] + + def build_checked_knowledge: (Hash[String, untyped] data) -> CheckedKnowledge + + def build_utterance: (Hash[String, untyped] data) -> Utterance + end +end diff --git a/test/test-jmrd.rb b/test/test-jmrd.rb new file mode 100644 index 00000000..82d1c45b --- /dev/null +++ b/test/test-jmrd.rb @@ -0,0 +1,151 @@ +class JMRDTest < Test::Unit::TestCase + sub_test_case("train") do + def setup + @dataset = Datasets::JMRD.new(type: :train) + end + + test("#each") do + assert_equal({ + :dialogue_size => 4575, + :dialog_id => "01884", + :movie_title => "時をかける少女", + :first_speaker => "recommender", + :recommender_q1 => 5, + :seeker_q1 => 4, + :knowledge_title => "時をかける少女", + :knowledge_year => "2006年", + :director_name => "細田守", + :utterance_size => 26, + :utterance_id => "01884_00", + :utterance_speaker => "recommender", + :utterance_text => "こんにちは", + :checked_knowledge_size => 1, + :checked_knowledge_type => "[知識なし]", + }, + first_dialogue_summary(@dataset.to_a)) + end + + test("knowledge keeps no knowledge option") do + dialogue = @dataset.first + assert_equal("[知識なし]", dialogue.knowledge.no_knowledge) + end + end + + sub_test_case("valid") do + def setup + @dataset = Datasets::JMRD.new(type: :valid) + end + + test("#each") do + assert_equal({ + :dialogue_size => 200, + :has_dialog_id => true, + :has_movie_title => true, + :valid_first_speaker => true, + }, + dialogue_summary(@dataset.to_a)) + end + end + + sub_test_case("test") do + def setup + @dataset = Datasets::JMRD.new(type: :test) + end + + test("#each") do + assert_equal({ + :dialogue_size => 300, + :has_dialog_id => true, + :has_movie_title => true, + :valid_first_speaker => true, + }, + dialogue_summary(@dataset.to_a)) + end + end + + sub_test_case("invalid type") do + test("raises error") do + message = ":type must be one of [:train, :valid, :test]: :invalid" + assert_raise(ArgumentError.new(message)) do + Datasets::JMRD.new(type: :invalid) + end + end + end + + sub_test_case("#metadata") do + def setup + @dataset = Datasets::JMRD.new(type: :train) + end + + test("#id") do + assert_equal("jmrd", @dataset.metadata.id) + end + + test("#name") do + assert_equal("Japanese Movie Recommendation Dialogue Dataset (JMRD)", + @dataset.metadata.name) + end + + test("#url") do + assert_equal("https://github.com/ku-nlp/JMRD", @dataset.metadata.url) + end + + test("#licenses") do + assert_equal([Datasets::License.new("CC-BY-SA-4.0")], + @dataset.metadata.licenses) + end + + test("#description") do + description = @dataset.metadata.description + assert do + [ + "Japanese Movie Recommendation Dialogue Dataset", + "5,000 dialogues", + "knowledge-grounded", + ].all? do |phrase| + description.include?(phrase) + end + end + end + end + + private + + def dialogue_summary(dialogues) + first_dialogue = dialogues[0] + { + :dialogue_size => dialogues.size, + :has_dialog_id => !first_dialogue.dialog_id.nil?, + :has_movie_title => !first_dialogue.movie_title.nil?, + :valid_first_speaker => valid_speaker?(first_dialogue.first_speaker), + } + end + + def valid_speaker?(speaker) + ["recommender", "seeker"].include?(speaker) + end + + def first_dialogue_summary(dialogues) + first_dialogue = dialogues[0] + first_utterance = first_dialogue.utterances[0] + first_checked_knowledge = first_utterance.checked_knowledge[0] + + { + :dialogue_size => dialogues.size, + :dialog_id => first_dialogue.dialog_id, + :movie_title => first_dialogue.movie_title, + :first_speaker => first_dialogue.first_speaker, + :recommender_q1 => first_dialogue.questionnaire.recommender.q1, + :seeker_q1 => first_dialogue.questionnaire.seeker.q1, + :knowledge_title => first_dialogue.knowledge.title, + :knowledge_year => first_dialogue.knowledge.year, + :director_name => first_dialogue.knowledge.director_name, + :utterance_size => first_dialogue.utterances.size, + :utterance_id => first_utterance.utterance_id, + :utterance_speaker => first_utterance.speaker, + :utterance_text => first_utterance.text, + :checked_knowledge_size => first_utterance.checked_knowledge.size, + :checked_knowledge_type => first_checked_knowledge.type, + } + end +end