diff --git a/diglab2ando.py b/diglab2bids.py similarity index 80% rename from diglab2ando.py rename to diglab2bids.py index 9d4d13c..ef4ea81 100644 --- a/diglab2ando.py +++ b/diglab2bids.py @@ -1,13 +1,15 @@ import re import pathlib import json -from ando.tools.generator.AnDOGenerator import AnDOData +from bep032tools.generator.BEP032Generator import BEP032Data from redcap_bridge.server_interface import download_records -from ando.checker import is_valid +from bep032tools.validator.BEP032Validator import is_valid + config_file = pathlib.Path(__file__).parent / 'config.json' project_name = 'SimpleProject' + with open(config_file) as f: conf = json.load(f) @@ -17,6 +19,7 @@ if not OUTPUT_FOLDER.exists(): OUTPUT_FOLDER.mkdir() + def get_metadata(conf, format): """ Fetch all recorded metadata from the server @@ -38,7 +41,7 @@ def get_metadata(conf, format): return records -def convert_to_bids(records, OUTPUT_FOLDER): +def convert_to_bids(records, files_per_record, OUTPUT_FOLDER): """ Parameters @@ -50,15 +53,14 @@ def convert_to_bids(records, OUTPUT_FOLDER): Returns ---------- """ - for record_dict in records: + for files, record_dict in zip(files_per_record, records): sub_id, ses_id = get_sub_ses_ids(record_dict) - gen = AnDOData(sub_id, ses_id, modality='ephys') - gen.register_data_files(get_data_file()) + gen = BEP032Data(sub_id, ses_id, modality='ephys') + gen.register_data_files(files) gen.basedir = OUTPUT_FOLDER - gen.generate_structure() - gen.generate_data_files(mode='move') - - generate_metadata_files(record_dict, gen.get_data_folder()) + gen.generate_directory_structure() + gen.generate_all_metadata_files(record_dict) + gen.organize_data_files() def get_sub_ses_ids(record_dict): @@ -89,47 +91,6 @@ def get_sub_ses_ids(record_dict): else: raise Exception("Record dict must only contain alphanumeric characters") -def get_data_file(): - """ - Parameters - ---------- - - Returns - ---------- - TODO: this needs to be replaced by a project-specific functions that converts the data to nix and provides the path to the nix file - """ - - dummy_nix_file = OUTPUT_FOLDER / 'dummy_file.nix' - if not dummy_nix_file.exists(): - dummy_nix_file.touch() - return dummy_nix_file - - -def generate_metadata_files(record_dict, save_dir): - """ - - Parameters - ---------- - record_dict: - save_dir: - - Returns - ---------- - - TODO: this needs to generate the basic BIDS metadata files and - """ - - # these can then be rearranged into the right location by the ando generator - filename = f'sub-{record_dict["guid"]}_ses-{record_dict["date"]}_ephys.json' - with open(save_dir / filename, 'w') as f: - json.dump(record_dict, f) - - metadata_filenames = ["dataset_description.json","tasks.json","participants.json" - "tasks.csv", "participants.csv"] - for filename in metadata_filenames: - with open(save_dir / filename, 'w') as f: - pass - if __name__ == '__main__': # json way of the world @@ -137,7 +98,7 @@ def generate_metadata_files(record_dict, save_dir): if not rec: raise ValueError(f'No records found for project {project_name}.') convert_to_bids(rec, OUTPUT_FOLDER) - ando.is_valid(OUTPUT_FOLDER) + is_valid(OUTPUT_FOLDER) diff --git a/setup.py b/setup.py index 5c6db21..1a96503 100644 --- a/setup.py +++ b/setup.py @@ -16,10 +16,10 @@ packages=find_packages(), author="Julia Sprenger, Jeremy Garcia", - description="diglab2ando is a tool that allows automatically creating a directory where data and metadata from a " - "neuroscientific experiment are stored, and that follows the AnDO (Animal Data Organization) " - "specifications ( https://int-nit.github.io/AnDO/ ), using as input a filled pdf form or a redcap " - "survey/form generated using the DigLaB tool used at INT", + description="diglab2bids is a tool that allows automatically creating a directory where data and metadata from a " + "neuroscientific experiment are stored, and that follows the DigLab " + "specifications (https://github.com/INT-NIT/DigLabTools/), using as input a filled pdf form or a redcap " + "survey/form", license='MIT', install_requires=[], include_package_data=True, diff --git a/tests/test_diglab2ando.py b/tests/test_diglab2bids.py similarity index 61% rename from tests/test_diglab2ando.py rename to tests/test_diglab2bids.py index b9c9a11..be01493 100644 --- a/tests/test_diglab2ando.py +++ b/tests/test_diglab2bids.py @@ -1,8 +1,10 @@ from unittest import TestCase - +from diglab2bids import convert_to_bids # Here for test on dependency class Test(TestCase): - def test_empty(self): + def test_convert_to_bids(self): + + self.assertEqual(True, True) diff --git a/tests/test_files/record.csv b/tests/test_files/record.csv new file mode 100644 index 0000000..6a77028 --- /dev/null +++ b/tests/test_files/record.csv @@ -0,0 +1,4 @@ +record_id,redcap_repeat_instrument,redcap_repeat_instance,redcap_survey_identifier,diglabform_timestamp,ethical_protocol_id,user,date,exp_name,guid,ses_number,modality___behaviour_eye,modality___behaviour_hand,modality___single_electrode,modality___multi_electrode,modality___emg,modality___int,modality___vsdi,modality___ecog,modality___seeg,stimulation,weight,comments_exp,comments_setup,comments_subject,data_quality,incomplete,subject_behaviour___very_motivated,subject_behaviour___working,subject_behaviour___thirsty,subject_behaviour___sleepy,subject_behaviour___unmotivated,subject_behaviour___agitated,subject_behaviour_multi,time_last_trial,subject_behaviour_2___agitated,subject_behaviour_2___resting,subject_behaviour_2___sleeping,subject_behaviour_2___working,reward_fluid,reward_fluid_additional,reward_fluid_type___water,reward_fluid_type___apple,reward_fluid_type___mixed,reward_fluid_type___other,reward_fluid_type_other,reward_other___fruit_fresh,reward_other___fruit_dry,reward_other___seeds,reward_other___treats,reward_other___insects,special_event_0,special_event_time_0,special_event_1,special_event_time_1,special_event_2,special_event_time_2,data_filename,exp_duration,task_name,task_mode,stim_set_primavoice,stim_set_morphing,stim_set_formant_saliency,stim_set_identity,stim_type_other,tone_frequency,tone_duration,stim_duration,stim_attenuation,stim_isi,stim_pretrial_delay,time_out_delay,reaction_timeout,min_n,max_n,reward_ratio,reward_duration,reward_volume,trial_count_correct,avg_stim_occurrence_correct,avg_stim_occurrence_all,tot_pres_stimuli_primavoice_correct,tot_pres_stimuli_primavoice_all,start_time_primavoice,tot_pres_stimuli_identity_correct,tot_pres_stimuli_identity_all,start_time_identity,tot_pres_stimuli_morphing_correct,tot_pres_stimuli_morphing_all,start_time_morphing,tot_pres_stimuli_formant_saliency_correct,tot_pres_stimuli_formant_saliency_all,start_time_formant_saliency,tot_pres_stimuli_bpn_correct,tot_pres_stimuli_bpn_all,start_time_bpn,tot_pres_stimuli_custom_stim_correct,tot_pres_stimuli_custom_stim_all,start_time_custom_stim,sua_1,sua_2,sua_3,sua_4,sua_5,sua_6,sua_7,sua_8,sua_9,sua_10,sua_11,sua_12,sua_13,sua_14,sua_15,sua_16,sua_17,sua_18,sua_19,sua_20,sua_21,sua_22,sua_23,sua_24,sua_25,sua_26,sua_27,sua_28,sua_29,sua_30,sua_31,sua_32,sua_33,sua_34,sua_35,sua_36,sua_37,sua_38,sua_39,sua_40,sua_41,sua_42,sua_43,sua_44,sua_45,sua_46,sua_47,sua_48,sua_49,sua_50,sua_51,sua_52,sua_53,sua_54,sua_55,sua_56,sua_57,sua_58,sua_59,sua_60,sua_61,sua_62,sua_63,sua_64,sua_65,sua_66,sua_67,sua_68,sua_69,sua_70,sua_71,sua_72,sua_73,sua_74,sua_75,sua_76,sua_77,sua_78,sua_79,sua_80,sua_81,sua_82,sua_83,sua_84,sua_85,sua_86,sua_87,sua_88,sua_89,sua_90,sua_91,sua_92,sua_93,sua_94,sua_95,sua_96,sua_isolated_1___isolated,sua_isolated_2___isolated,sua_isolated_3___isolated,sua_isolated_4___isolated,sua_isolated_5___isolated,sua_isolated_6___isolated,sua_isolated_7___isolated,sua_isolated_8___isolated,sua_isolated_9___isolated,sua_isolated_10___isolated,sua_isolated_11___isolated,sua_isolated_12___isolated,sua_isolated_13___isolated,sua_isolated_14___isolated,sua_isolated_15___isolated,sua_isolated_16___isolated,sua_isolated_17___isolated,sua_isolated_18___isolated,sua_isolated_19___isolated,sua_isolated_20___isolated,sua_isolated_21___isolated,sua_isolated_22___isolated,sua_isolated_23___isolated,sua_isolated_24___isolated,sua_isolated_25___isolated,sua_isolated_26___isolated,sua_isolated_27___isolated,sua_isolated_28___isolated,sua_isolated_29___isolated,sua_isolated_30___isolated,sua_isolated_31___isolated,sua_isolated_32___isolated,sua_isolated_33___isolated,sua_isolated_34___isolated,sua_isolated_35___isolated,sua_isolated_36___isolated,sua_isolated_37___isolated,sua_isolated_38___isolated,sua_isolated_39___isolated,sua_isolated_40___isolated,sua_isolated_41___isolated,sua_isolated_42___isolated,sua_isolated_43___isolated,sua_isolated_44___isolated,sua_isolated_45___isolated,sua_isolated_46___isolated,sua_isolated_47___isolated,sua_isolated_48___isolated,sua_isolated_49___isolated,sua_isolated_50___isolated,sua_isolated_51___isolated,sua_isolated_52___isolated,sua_isolated_53___isolated,sua_isolated_54___isolated,sua_isolated_55___isolated,sua_isolated_56___isolated,sua_isolated_57___isolated,sua_isolated_58___isolated,sua_isolated_59___isolated,sua_isolated_60___isolated,sua_isolated_61___isolated,sua_isolated_62___isolated,sua_isolated_63___isolated,sua_isolated_64___isolated,sua_isolated_65___isolated,sua_isolated_66___isolated,sua_isolated_67___isolated,sua_isolated_68___isolated,sua_isolated_69___isolated,sua_isolated_70___isolated,sua_isolated_71___isolated,sua_isolated_72___isolated,sua_isolated_73___isolated,sua_isolated_74___isolated,sua_isolated_75___isolated,sua_isolated_76___isolated,sua_isolated_77___isolated,sua_isolated_78___isolated,sua_isolated_79___isolated,sua_isolated_80___isolated,sua_isolated_81___isolated,sua_isolated_82___isolated,sua_isolated_83___isolated,sua_isolated_84___isolated,sua_isolated_85___isolated,sua_isolated_86___isolated,sua_isolated_87___isolated,sua_isolated_88___isolated,sua_isolated_89___isolated,sua_isolated_90___isolated,sua_isolated_91___isolated,sua_isolated_92___isolated,sua_isolated_93___isolated,sua_isolated_94___isolated,sua_isolated_95___isolated,sua_isolated_96___isolated,diglabform_complete +1,diglabform,1,,2022-01-14 17:55:00,0000_2000000000000000_v1,sprenger.j,2021-12-07,myexperiment,AAA11111111,1,0,1,0,1,0,0,0,0,0,,5.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,115,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211207-142257,46,mytask1,task_active,training96,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,306,,,1532,1773,,,,,,,,,,,,,,,,,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,1,1,0,1,1,0,1,0,1,0,0,1,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 +2,diglabform,1,,2022-01-14 18:38:50,0000_2000000000000000_v2,sprenger.j,2021-12-07,myexperiment,AAA11111111,2,0,1,0,1,0,0,0,0,0,,6.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,130,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211208-142020,44,mytask2,task_active,testing,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,335,,,1692,1866,,,,,,,,,,,,,,,,,1,0,1,1,0,0,1,0,1,1,1,0,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,0,1,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,1,1,0,1,1,1,1,1,0,0,1,1,0,1,1,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 +3,diglabform,1,,2022-01-14 18:38:50,0000_2000000000000000_v2,sprenger.j,2021-12-08,myexperiment,AAA11111111,1,0,1,0,1,0,0,0,0,0,,6.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,130,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211208-142020,44,mytask2,task_active,testing,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,335,,,1692,1866,,,,,,,,,,,,,,,,,1,0,1,1,0,0,1,0,1,1,1,0,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,0,1,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,1,1,0,1,1,1,1,1,0,0,1,1,0,1,1,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2 diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 0000000..8cfcdcb --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,31 @@ +import os +import tempfile +import shutil +import pathlib +test_directory = pathlib.Path(tempfile.gettempdir()) / 'diglab2bids_testfiles' + + +def initialize_test_directory(clean=True): + """ + Create main test folder if required + + Parameters + ---------- + clean: (bool) + Remove test folder first in case it exists. + + Returns + ------- + test_directory: (str) + path of the test directory + """ + if clean and os.path.exists(test_directory): + shutil.rmtree(test_directory) + + if not os.path.exists(test_directory): + os.mkdir(test_directory) + packaged_testfolder = pathlib.Path(__file__).parent / 'test_files' + shutil.copytree(packaged_testfolder, test_directory / 'test_files') + + return test_directory +