|
| 1 | +#!/usr/bin/env python |
| 2 | +# -*- coding: utf-8 -*- |
| 3 | +# |
| 4 | +# Copyright 2022 Telefónica Soluciones de Informática y Comunicaciones de España, S.A.U. |
| 5 | +# |
| 6 | +# This file is part of tc_etl_lib |
| 7 | +# |
| 8 | +# tc_etl_lib is free software: you can redistribute it and/or |
| 9 | +# modify it under the terms of the GNU Affero General Public License as |
| 10 | +# published by the Free Software Foundation, either version 3 of the |
| 11 | +# License, or (at your option) any later version. |
| 12 | +# |
| 13 | +# tc_etl_lib is distributed in the hope that it will be useful, |
| 14 | +# but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 15 | +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero |
| 16 | +# General Public License for more details. |
| 17 | +# |
| 18 | +# You should have received a copy of the GNU Affero General Public License |
| 19 | +# along with IoT orchestrator. If not, see http://www.gnu.org/licenses/. |
| 20 | + |
| 21 | +""" |
| 22 | +Minio routines for Python: |
| 23 | + - minioManager. |
| 24 | +""" |
| 25 | +from minio import Minio |
| 26 | +from typing import Optional, cast |
| 27 | +import logging |
| 28 | + |
| 29 | +logger = logging.getLogger(__name__) |
| 30 | + |
| 31 | + |
| 32 | +class minioManager: |
| 33 | + """Minio Manager |
| 34 | +
|
| 35 | + endpoint: define minio endpoint |
| 36 | + access_key: user to log in to minio |
| 37 | + secret_key: password to log in to minio |
| 38 | + secure: flag to select if the connection to MinIO is https or http (True by default) |
| 39 | + client: authenticated MinIO client |
| 40 | + """ |
| 41 | + endpoint: str |
| 42 | + access_key: str |
| 43 | + secret_key: str |
| 44 | + secure: bool |
| 45 | + client: Minio |
| 46 | + |
| 47 | + def __init__(self, endpoint: Optional[str] = None, access_key: Optional[str] = None, secret_key: Optional[str] = None, secure=True): |
| 48 | + |
| 49 | + messageError = [] |
| 50 | + if endpoint is None: |
| 51 | + messageError.append('<<endpoint>>') |
| 52 | + |
| 53 | + if access_key is None: |
| 54 | + messageError.append('<<access_key>>') |
| 55 | + |
| 56 | + if secret_key is None: |
| 57 | + messageError.append('<<secret_key>>') |
| 58 | + |
| 59 | + if len(messageError) != 0: |
| 60 | + defineParams = messageError[0] |
| 61 | + if len(messageError) != 1: |
| 62 | + defineParams = " and ".join( |
| 63 | + [", ".join(messageError[:-1]), messageError[-1]]) |
| 64 | + raise ValueError(f'You must define {defineParams} in minioManager') |
| 65 | + |
| 66 | + # At this point, all Optional[str] have been validated to be not None. |
| 67 | + # cast them to let type checker knows. |
| 68 | + self.endpoint = cast(str, endpoint) |
| 69 | + self.access_key = cast(str, access_key) |
| 70 | + self.secret_key = cast(str, secret_key) |
| 71 | + self.secure = secure |
| 72 | + self.client = self.__init_client() |
| 73 | + |
| 74 | + def __init_client(self): |
| 75 | + """ |
| 76 | + Create a MinIO client with the class endpoint, its access key and secret key. |
| 77 | +
|
| 78 | + :return authenticated MinIO client |
| 79 | + """ |
| 80 | + return Minio( |
| 81 | + self.endpoint, |
| 82 | + self.access_key, |
| 83 | + self.secret_key, |
| 84 | + secure=self.secure |
| 85 | + ) |
| 86 | + |
| 87 | + def create_bucket(self, bucket_name): |
| 88 | + """ |
| 89 | + Create the bucket if it doesn't exist. |
| 90 | +
|
| 91 | + :param bucket_name: name of the bucket where the file is located |
| 92 | + """ |
| 93 | + found = self.client.bucket_exists(bucket_name) |
| 94 | + if not found: |
| 95 | + self.client.make_bucket(bucket_name) |
| 96 | + logger.debug(f'Created bucket ({bucket_name})') |
| 97 | + else: |
| 98 | + logger.debug(f'Bucket {bucket_name} already exists') |
| 99 | + |
| 100 | + def remove_bucket(self, bucket_name): |
| 101 | + """ |
| 102 | + Remove the bucket if it exists. |
| 103 | +
|
| 104 | + :param bucket_name: name of the bucket where the file is located |
| 105 | + """ |
| 106 | + found = self.client.bucket_exists(bucket_name) |
| 107 | + if found: |
| 108 | + self.client.remove_bucket(bucket_name) |
| 109 | + logger.debug(f'Removed bucket {bucket_name}') |
| 110 | + else: |
| 111 | + logger.debug(f'Bucket {bucket_name} doesnt exist') |
| 112 | + |
| 113 | + def upload_file(self, bucket_name, destination_file, source_file): |
| 114 | + """ |
| 115 | + Upload the file, renaming it in the process |
| 116 | +
|
| 117 | + :param bucket_name: name of the bucket where the file is located |
| 118 | + :param destination_file: name of the file to retrieve (can include path without bucket_name) |
| 119 | + :param source_file: name of the file to upload (can include path) |
| 120 | + :return object with the status of the upload |
| 121 | + """ |
| 122 | + # Bucket must exist before uploading file |
| 123 | + self.create_bucket(bucket_name) |
| 124 | + |
| 125 | + logger.debug( |
| 126 | + f'Uploading {source_file} as object {destination_file} to bucket {bucket_name}') |
| 127 | + return self.client.fput_object( |
| 128 | + bucket_name, |
| 129 | + object_name=destination_file, |
| 130 | + file_path=source_file, |
| 131 | + ) |
| 132 | + |
| 133 | + def process_file(self, bucket_name, file, processing_method, chunk_size=500000): |
| 134 | + """Retrieves a file in chunks and applies a function to each chunk |
| 135 | +
|
| 136 | + :param bucket_name: name of the bucket where the file is located |
| 137 | + :param file: name of the file to retrieve (can include path without bucket_name) |
| 138 | + :param processing_method: method to apply to each chunk of the retrieved file |
| 139 | + :param chunk_size: size in bytes of the chunks to retrieve (500000 by default) |
| 140 | + """ |
| 141 | + file_size = self.client.stat_object( |
| 142 | + bucket_name, object_name=file).size or 0 |
| 143 | + |
| 144 | + response = None |
| 145 | + for offset in range(0, file_size, chunk_size): |
| 146 | + # Get the file |
| 147 | + try: |
| 148 | + response = self.client.get_object( |
| 149 | + bucket_name, file, offset, chunk_size) |
| 150 | + # response.data returns bytes |
| 151 | + processing_method(response.data) |
| 152 | + except Exception as e: |
| 153 | + raise Exception( |
| 154 | + f'An error occured while processing the file: {e}') |
| 155 | + |
| 156 | + logger.debug(f'Processing ended.') |
| 157 | + if response: |
| 158 | + response.close() |
| 159 | + response.release_conn() |
0 commit comments