Allow Datapoint/Segmentations upload via Frontend

backend/routes/data.py

@@ -1,8 +1,10 @@
+import shutil
 import json
 import sqlalchemy as sa
 import uuid
 
 from pathlib import Path
+from zipfile import ZipFile
 
 from flask import jsonify, flash, redirect, url_for, request, send_from_directory
 from flask_jwt_extended import jwt_required, get_jwt_identity
@@ -15,19 +17,22 @@
 
 from . import api
 
+ALLOWED_COMPRESSED_EXTENSIONS = ["zip"]
+ALLOWED_ANNOTATION_EXTENSIONS = ["json"]
 ALLOWED_EXTENSIONS = ["wav", "mp3", "ogg"]
 
-
 @api.route("/audio/<path:file_name>", methods=["GET"])
 @jwt_required
 def send_audio_file(file_name):
     return send_from_directory(app.config["UPLOAD_FOLDER"], file_name)
 
 
-def validate_segmentation(segment):
+def validate_segmentation(segment, without_data=False):
     """Validate the segmentation before accepting the annotation's upload from users
     """
     required_key = {"start_time", "end_time", "transcription"}
+    if without_data:
+        required_key.add("filename") # requried to search the datapoint
 
     if set(required_key).issubset(segment.keys()):
         return True
@@ -190,3 +195,103 @@ def add_data():
         ),
         201,
     )
+
+# TODO: Add authentication
+@api.route("/datazip", methods=["POST"])
+# @jwt_required
+def add_datazip():
+    # identity = get_jwt_identity()
+    # request_user = User.query.filter_by(username=identity["username"]).first()
+    # app.logger.info(f"Current user is: {request_user}")
+    # is_admin = True if request_user.role.role == "admin" else False
+
+    # if is_admin == False:
+    #     return jsonify(message="Unauthorized access!"), 401
+
+    # if not request.is_json:
+    #     return jsonify(message="Missing JSON in request"), 400
+
+    pid = request.form.get("projectId", None)
+    username = request.form.get("userId", None)
+    project = Project.query.filter_by(id=pid).first()
+    user = User.query.filter_by(username=username).first()
+
+    files = request.files.items()
+
+    app.logger.info(f"user: {user} / project: {project}")
+
+    for _, audio_file in files:
+        filename = audio_file.filename
+        filename_ext = Path(filename).suffix.lower()
+        if filename_ext[1:] in ALLOWED_COMPRESSED_EXTENSIONS:
+            app.logger.info(f"Compressed file in consideration: {filename}")
+            with ZipFile(audio_file, "r") as zip_obj:
+                for cmprsd_filename in zip_obj.namelist():
+                    app.logger.info(f"cmpressed files are: {cmprsd_filename}")
+                    cmprsd_extension = Path(cmprsd_filename).suffix.lower()
+                    if cmprsd_extension[1:] in ALLOWED_EXTENSIONS:
+                        zip_obj.extract(
+                            cmprsd_filename,
+                            app.config["UPLOAD_FOLDER"]
+                        )
+                        tfilename = f"{str(uuid.uuid4().hex)}{cmprsd_extension}"
+                        shutil.move(
+                            Path(app.config["UPLOAD_FOLDER"]).joinpath(cmprsd_filename), Path(
+                                app.config["UPLOAD_FOLDER"]).joinpath(tfilename)
+                        )
+                        data = Data(
+                            project_id=project.id,
+                            filename=tfilename,
+                            original_filename=cmprsd_filename,
+                            reference_transcription="",
+                            is_marked_for_review=True,
+                            assigned_user_id=user.id,
+                        )
+
+
+                        db.session.add(data)
+                        db.session.flush()
+                        db.session.commit()
+
+                    elif cmprsd_extension[1:] in ALLOWED_ANNOTATION_EXTENSIONS:
+                        pass
+                        # reading the file, temp store it
+                        temp_loc = Path(app.config["UPLOAD_FOLDER"]
+                                        ).joinpath(cmprsd_filename)
+                        zip_obj.extract(cmprsd_filename,
+                                        app.config["UPLOAD_FOLDER"])
+
+                        segmentations = json.load(open(temp_loc, "r"))
+
+                        for _segment in segmentations:
+                            validated = validate_segmentation(_segment, without_data=True)
+
+                            if not validated:
+                                continue  # skip this datapoint
+
+                            if validated:
+                                try:
+                                    data = Data.query.filter_by(
+                                        project_id=pid, original_filename=_segment['filename']).first()
+
+                                    if data.id:
+
+                                        new_segment = generate_segmentation(
+                                            data_id=data.id,
+                                            project_id=project.id,
+                                            end_time=_segment["end_time"],
+                                            start_time=_segment["start_time"],
+                                            transcription=_segment["transcription"],
+                                            annotations=_segment.get("annotations", {})
+                                        )
+
+                                        data.set_segmentations([new_segment])
+                                        app.logger.info(f"new_segment: {new_segment.data_id}")
+                                        db.session.commit()
+                                        db.session.refresh(data)
+                                except Exception as e:
+                                    app.logger.info(f"Error {e} for data: {data.id}")
+
+    return jsonify(
+        resp="HAHA"
+    )

frontend/src/containers/forms/uploadDataForm.js

@@ -0,0 +1,46 @@
+import React from 'react';
+import { withRouter } from "react-router";
+import Dropzone from 'react-dropzone-uploader';
+import { withStore } from "@spyna/react-store";
+import 'react-dropzone-uploader/dist/styles.css'
+
+class UploadDataForm extends React.Component {
+    constructor(props) {
+        super(props);
+
+        const projectId = this.props.projectId;
+        const userId = this.props.userId;
+
+        this.initialState = {
+            userId,
+            projectId,
+            addDataUrl: `/api/datazip`,
+        };
+
+        this.state = Object.assign({}, this.initialState);
+        console.log(this.props);
+    }
+
+    getUploadParams = ({ file, meta }) => {
+        const body = new FormData()
+        body.append('fileField', file)
+        body.append('userId', this.state.userId)
+        body.append('projectId', this.state.projectId)
+        return { url: this.state.addDataUrl, body }
+    }
+    handleChangeStatus = ({ meta, file }, status) => { console.log(status, meta, file) }
+
+    render() {
+        return (
+            <Dropzone
+                getUploadParams={this.getUploadParams}
+                onChangeStatus={this.handleChangeStatus}
+                accept="application/zip,application/x-zip,application/x-zip-compressed,application/octet-stream"
+            // accept="image/*,audio/*,video/*"
+            />
+        )
+    }
+
+}
+
+export default withStore(withRouter(UploadDataForm));

frontend/src/containers/modal.js

@@ -2,6 +2,7 @@ import React from "react";
 import Modal from "react-bootstrap/Modal";
 
 import CreateUserForm from "./forms/createUserForm";
+import UploadDataForm from "./forms/uploadDataForm";
 import EditUserForm from "./forms/editUserForm";
 import CreateProjectForm from "./forms/createProjectForm";
 import CreateLabelForm from "./forms/createLabelForm";
@@ -28,6 +29,9 @@ const FormModal = (props) => {
       <Modal.Body>
         {props.formType === "NEW_USER" ? <CreateUserForm /> : null}
         {props.formType === "NEW_PROJECT" ? <CreateProjectForm /> : null}
+        {props.formType === "NEW_DATA" ? (
+          <UploadDataForm projectId={props.projectId} userId={props.userId} />
+        ) : null}
         {props.formType === "EDIT_USER" ? (
           <EditUserForm userId={props.userId} />
         ) : null}

frontend/src/pages/admin.js

@@ -9,6 +9,7 @@ import {
   faUserPlus,
   faTags,
   faDownload,
+  faUpload,
 } from "@fortawesome/free-solid-svg-icons";
 import { IconButton } from "../components/button";
 import Loader from "../components/loader";
@@ -165,6 +166,11 @@ class Admin extends React.Component {
       });
   }
 
+  handleUploadData(e, userId, projectId) {
+    this.setModalShow(true);
+    this.setState({ formType: "NEW_DATA", title: "Create New Project", userId, projectId });
+  }
+
   setModalShow(modalShow) {
     this.setState({ modalShow });
   }
@@ -260,6 +266,18 @@ class Admin extends React.Component {
                                 )
                               }
                             />
+                            <IconButton
+                              icon={faUpload}
+                              size="sm"
+                              title={"Upload data"}
+                              onClick={(e) =>
+                                this.handleUploadData(
+                                  e,
+                                  "admin",
+                                  project["project_id"]
+                                )
+                              }
+                            />
                             <IconButton
                               icon={faDownload}
                               size="sm"