minor updates

diglab2ando.py → diglab2bids.py

@@ -1,13 +1,15 @@
 import re
 import pathlib
 import json
-from ando.tools.generator.AnDOGenerator import AnDOData
+from bep032tools.generator.BEP032Generator import BEP032Data
 from redcap_bridge.server_interface import download_records
-from ando.checker import is_valid
+from bep032tools.validator.BEP032Validator import is_valid
+
 
 config_file = pathlib.Path(__file__).parent / 'config.json'
 project_name = 'SimpleProject'
 
+
 with open(config_file) as f:
     conf = json.load(f)
 
@@ -17,6 +19,7 @@
 if not OUTPUT_FOLDER.exists():
     OUTPUT_FOLDER.mkdir()
 
+
 def get_metadata(conf, format):
     """
     Fetch all recorded metadata from the server
@@ -38,7 +41,7 @@ def get_metadata(conf, format):
 
     return records
 
-def convert_to_bids(records, OUTPUT_FOLDER):
+def convert_to_bids(records, files_per_record, OUTPUT_FOLDER):
     """
 
     Parameters
@@ -50,15 +53,14 @@ def convert_to_bids(records, OUTPUT_FOLDER):
     Returns
     ----------
     """
-    for record_dict in records:
+    for files, record_dict in zip(files_per_record, records):
         sub_id, ses_id = get_sub_ses_ids(record_dict)
-        gen = AnDOData(sub_id, ses_id, modality='ephys')
-        gen.register_data_files(get_data_file())
+        gen = BEP032Data(sub_id, ses_id, modality='ephys')
+        gen.register_data_files(files)
         gen.basedir = OUTPUT_FOLDER
-        gen.generate_structure()
-        gen.generate_data_files(mode='move')
-
-        generate_metadata_files(record_dict, gen.get_data_folder())
+        gen.generate_directory_structure()
+        gen.generate_all_metadata_files(record_dict)
+        gen.organize_data_files()
 
 
 def get_sub_ses_ids(record_dict):
@@ -89,55 +91,14 @@ def get_sub_ses_ids(record_dict):
     else:
         raise Exception("Record dict must only contain alphanumeric characters")
 
-def get_data_file():
-    """
-    Parameters
-    ----------
-
-    Returns
-    ----------
-    TODO: this needs to be replaced by a project-specific functions that converts the data to nix and provides the path to the nix file
-    """
-
-    dummy_nix_file = OUTPUT_FOLDER / 'dummy_file.nix'
-    if not dummy_nix_file.exists():
-        dummy_nix_file.touch()
-    return dummy_nix_file
-
-
-def generate_metadata_files(record_dict, save_dir):
-    """
-
-    Parameters
-    ----------
-    record_dict:
-    save_dir:
-
-    Returns
-    ----------
-
-    TODO: this needs to generate the basic BIDS metadata files and
-    """
-
-    # these can then be rearranged into the right location by the ando generator
-    filename = f'sub-{record_dict["guid"]}_ses-{record_dict["date"]}_ephys.json'
-    with open(save_dir / filename, 'w') as f:
-        json.dump(record_dict, f)
-
-    metadata_filenames = ["dataset_description.json","tasks.json","participants.json"
-                          "tasks.csv", "participants.csv"]
-    for filename in metadata_filenames:
-        with open(save_dir / filename, 'w') as f:
-          pass
-
 
 if __name__ == '__main__':
     # json way of the world
     rec = get_metadata(PROJECT_DEF, 'json')
     if not rec:
         raise ValueError(f'No records found for project {project_name}.')
     convert_to_bids(rec, OUTPUT_FOLDER)
-    ando.is_valid(OUTPUT_FOLDER)
+    is_valid(OUTPUT_FOLDER)
 
 
 

setup.py

@@ -16,10 +16,10 @@
     packages=find_packages(),
 
     author="Julia Sprenger, Jeremy Garcia",
-    description="diglab2ando is a tool that allows automatically creating a directory where data and metadata from a "
-                "neuroscientific experiment are stored, and that follows the AnDO (Animal Data Organization) "
-                "specifications ( https://int-nit.github.io/AnDO/ ), using as input a filled pdf form or a redcap "
-                "survey/form generated using the DigLaB tool used at INT",
+    description="diglab2bids is a tool that allows automatically creating a directory where data and metadata from a "
+                "neuroscientific experiment are stored, and that follows the DigLab "
+                "specifications (https://github.com/INT-NIT/DigLabTools/), using as input a filled pdf form or a redcap "
+                "survey/form",
     license='MIT',
     install_requires=[],
     include_package_data=True,

tests/test_diglab2ando.py → tests/test_diglab2bids.py

@@ -1,8 +1,10 @@
 from unittest import TestCase
-
+from diglab2bids import convert_to_bids
 
 # Here for test on dependency
 
 class Test(TestCase):
-    def test_empty(self):
+    def test_convert_to_bids(self):
+
+
         self.assertEqual(True, True)

tests/test_files/record.csv

@@ -0,0 +1,4 @@
+record_id,redcap_repeat_instrument,redcap_repeat_instance,redcap_survey_identifier,diglabform_timestamp,ethical_protocol_id,user,date,exp_name,guid,ses_number,modality___behaviour_eye,modality___behaviour_hand,modality___single_electrode,modality___multi_electrode,modality___emg,modality___int,modality___vsdi,modality___ecog,modality___seeg,stimulation,weight,comments_exp,comments_setup,comments_subject,data_quality,incomplete,subject_behaviour___very_motivated,subject_behaviour___working,subject_behaviour___thirsty,subject_behaviour___sleepy,subject_behaviour___unmotivated,subject_behaviour___agitated,subject_behaviour_multi,time_last_trial,subject_behaviour_2___agitated,subject_behaviour_2___resting,subject_behaviour_2___sleeping,subject_behaviour_2___working,reward_fluid,reward_fluid_additional,reward_fluid_type___water,reward_fluid_type___apple,reward_fluid_type___mixed,reward_fluid_type___other,reward_fluid_type_other,reward_other___fruit_fresh,reward_other___fruit_dry,reward_other___seeds,reward_other___treats,reward_other___insects,special_event_0,special_event_time_0,special_event_1,special_event_time_1,special_event_2,special_event_time_2,data_filename,exp_duration,task_name,task_mode,stim_set_primavoice,stim_set_morphing,stim_set_formant_saliency,stim_set_identity,stim_type_other,tone_frequency,tone_duration,stim_duration,stim_attenuation,stim_isi,stim_pretrial_delay,time_out_delay,reaction_timeout,min_n,max_n,reward_ratio,reward_duration,reward_volume,trial_count_correct,avg_stim_occurrence_correct,avg_stim_occurrence_all,tot_pres_stimuli_primavoice_correct,tot_pres_stimuli_primavoice_all,start_time_primavoice,tot_pres_stimuli_identity_correct,tot_pres_stimuli_identity_all,start_time_identity,tot_pres_stimuli_morphing_correct,tot_pres_stimuli_morphing_all,start_time_morphing,tot_pres_stimuli_formant_saliency_correct,tot_pres_stimuli_formant_saliency_all,start_time_formant_saliency,tot_pres_stimuli_bpn_correct,tot_pres_stimuli_bpn_all,start_time_bpn,tot_pres_stimuli_custom_stim_correct,tot_pres_stimuli_custom_stim_all,start_time_custom_stim,sua_1,sua_2,sua_3,sua_4,sua_5,sua_6,sua_7,sua_8,sua_9,sua_10,sua_11,sua_12,sua_13,sua_14,sua_15,sua_16,sua_17,sua_18,sua_19,sua_20,sua_21,sua_22,sua_23,sua_24,sua_25,sua_26,sua_27,sua_28,sua_29,sua_30,sua_31,sua_32,sua_33,sua_34,sua_35,sua_36,sua_37,sua_38,sua_39,sua_40,sua_41,sua_42,sua_43,sua_44,sua_45,sua_46,sua_47,sua_48,sua_49,sua_50,sua_51,sua_52,sua_53,sua_54,sua_55,sua_56,sua_57,sua_58,sua_59,sua_60,sua_61,sua_62,sua_63,sua_64,sua_65,sua_66,sua_67,sua_68,sua_69,sua_70,sua_71,sua_72,sua_73,sua_74,sua_75,sua_76,sua_77,sua_78,sua_79,sua_80,sua_81,sua_82,sua_83,sua_84,sua_85,sua_86,sua_87,sua_88,sua_89,sua_90,sua_91,sua_92,sua_93,sua_94,sua_95,sua_96,sua_isolated_1___isolated,sua_isolated_2___isolated,sua_isolated_3___isolated,sua_isolated_4___isolated,sua_isolated_5___isolated,sua_isolated_6___isolated,sua_isolated_7___isolated,sua_isolated_8___isolated,sua_isolated_9___isolated,sua_isolated_10___isolated,sua_isolated_11___isolated,sua_isolated_12___isolated,sua_isolated_13___isolated,sua_isolated_14___isolated,sua_isolated_15___isolated,sua_isolated_16___isolated,sua_isolated_17___isolated,sua_isolated_18___isolated,sua_isolated_19___isolated,sua_isolated_20___isolated,sua_isolated_21___isolated,sua_isolated_22___isolated,sua_isolated_23___isolated,sua_isolated_24___isolated,sua_isolated_25___isolated,sua_isolated_26___isolated,sua_isolated_27___isolated,sua_isolated_28___isolated,sua_isolated_29___isolated,sua_isolated_30___isolated,sua_isolated_31___isolated,sua_isolated_32___isolated,sua_isolated_33___isolated,sua_isolated_34___isolated,sua_isolated_35___isolated,sua_isolated_36___isolated,sua_isolated_37___isolated,sua_isolated_38___isolated,sua_isolated_39___isolated,sua_isolated_40___isolated,sua_isolated_41___isolated,sua_isolated_42___isolated,sua_isolated_43___isolated,sua_isolated_44___isolated,sua_isolated_45___isolated,sua_isolated_46___isolated,sua_isolated_47___isolated,sua_isolated_48___isolated,sua_isolated_49___isolated,sua_isolated_50___isolated,sua_isolated_51___isolated,sua_isolated_52___isolated,sua_isolated_53___isolated,sua_isolated_54___isolated,sua_isolated_55___isolated,sua_isolated_56___isolated,sua_isolated_57___isolated,sua_isolated_58___isolated,sua_isolated_59___isolated,sua_isolated_60___isolated,sua_isolated_61___isolated,sua_isolated_62___isolated,sua_isolated_63___isolated,sua_isolated_64___isolated,sua_isolated_65___isolated,sua_isolated_66___isolated,sua_isolated_67___isolated,sua_isolated_68___isolated,sua_isolated_69___isolated,sua_isolated_70___isolated,sua_isolated_71___isolated,sua_isolated_72___isolated,sua_isolated_73___isolated,sua_isolated_74___isolated,sua_isolated_75___isolated,sua_isolated_76___isolated,sua_isolated_77___isolated,sua_isolated_78___isolated,sua_isolated_79___isolated,sua_isolated_80___isolated,sua_isolated_81___isolated,sua_isolated_82___isolated,sua_isolated_83___isolated,sua_isolated_84___isolated,sua_isolated_85___isolated,sua_isolated_86___isolated,sua_isolated_87___isolated,sua_isolated_88___isolated,sua_isolated_89___isolated,sua_isolated_90___isolated,sua_isolated_91___isolated,sua_isolated_92___isolated,sua_isolated_93___isolated,sua_isolated_94___isolated,sua_isolated_95___isolated,sua_isolated_96___isolated,diglabform_complete
+1,diglabform,1,,2022-01-14 17:55:00,0000_2000000000000000_v1,sprenger.j,2021-12-07,myexperiment,AAA11111111,1,0,1,0,1,0,0,0,0,0,,5.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,115,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211207-142257,46,mytask1,task_active,training96,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,306,,,1532,1773,,,,,,,,,,,,,,,,,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,1,0,1,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,1,1,1,0,1,1,0,1,0,1,0,0,1,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2
+2,diglabform,1,,2022-01-14 18:38:50,0000_2000000000000000_v2,sprenger.j,2021-12-07,myexperiment,AAA11111111,2,0,1,0,1,0,0,0,0,0,,6.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,130,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211208-142020,44,mytask2,task_active,testing,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,335,,,1692,1866,,,,,,,,,,,,,,,,,1,0,1,1,0,0,1,0,1,1,1,0,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,0,1,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,1,1,0,1,1,1,1,1,0,0,1,1,0,1,1,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2
+3,diglabform,1,,2022-01-14 18:38:50,0000_2000000000000000_v2,sprenger.j,2021-12-08,myexperiment,AAA11111111,1,0,1,0,1,0,0,0,0,0,,6.7,,,,good,,1,0,0,0,0,0,,,0,0,0,0,130,0,0,0,1,0,,1,1,0,1,0,,,,,,,data-211208-142020,44,mytask2,task_active,testing,,,,,1000,500,500,0,250,1500,4000,250,3,7,2,340,0.48,335,,,1692,1866,,,,,,,,,,,,,,,,,1,0,1,1,0,0,1,0,1,1,1,0,1,1,1,1,0,1,1,1,1,1,0,1,1,1,1,1,0,1,1,1,0,0,1,1,1,1,1,0,1,0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,1,1,1,0,1,1,1,1,0,1,1,0,1,0,1,1,0,1,1,1,1,1,0,0,1,1,0,1,1,0,0,0,1,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2

tests/utils.py

@@ -0,0 +1,31 @@
+import os
+import tempfile
+import shutil
+import pathlib
+test_directory = pathlib.Path(tempfile.gettempdir()) / 'diglab2bids_testfiles'
+
+
+def initialize_test_directory(clean=True):
+    """
+    Create main test folder if required
+
+    Parameters
+    ----------
+    clean: (bool)
+        Remove test folder first in case it exists.
+
+    Returns
+    -------
+    test_directory: (str)
+        path of the test directory
+    """
+    if clean and os.path.exists(test_directory):
+        shutil.rmtree(test_directory)
+
+    if not os.path.exists(test_directory):
+        os.mkdir(test_directory)
+    packaged_testfolder = pathlib.Path(__file__).parent / 'test_files'
+    shutil.copytree(packaged_testfolder, test_directory / 'test_files')
+
+    return test_directory
+