View Javadoc
1   /*
2    * Copyright (c) 2002-2024, City of Paris
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without
6    * modification, are permitted provided that the following conditions
7    * are met:
8    *
9    *  1. Redistributions of source code must retain the above copyright notice
10   *     and the following disclaimer.
11   *
12   *  2. Redistributions in binary form must reproduce the above copyright notice
13   *     and the following disclaimer in the documentation and/or other materials
14   *     provided with the distribution.
15   *
16   *  3. Neither the name of 'Mairie de Paris' nor 'Lutece' nor the names of its
17   *     contributors may be used to endorse or promote products derived from
18   *     this software without specific prior written permission.
19   *
20   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
24   * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   * POSSIBILITY OF SUCH DAMAGE.
31   *
32   * License 1.0
33   */
34  
35  package fr.paris.lutece.plugins.strois.service;
36  
37  import com.google.common.collect.Multimap;
38  import fr.paris.lutece.plugins.strois.util.S3Util;
39  import fr.paris.lutece.portal.service.util.AppLogService;
40  import fr.paris.lutece.portal.service.util.AppPropertiesService;
41  import io.minio.GetObjectArgs;
42  import io.minio.GetObjectTagsArgs;
43  import io.minio.MinioClient;
44  import io.minio.ObjectWriteResponse;
45  import io.minio.PutObjectArgs;
46  import io.minio.RemoveObjectArgs;
47  import io.minio.SnowballObject;
48  import io.minio.UploadSnowballObjectsArgs;
49  import io.minio.errors.ErrorResponseException;
50  import io.minio.errors.InsufficientDataException;
51  import io.minio.errors.InternalException;
52  import io.minio.errors.InvalidResponseException;
53  import io.minio.errors.MinioException;
54  import io.minio.errors.ServerException;
55  import io.minio.errors.XmlParserException;
56  import io.minio.messages.Tags;
57  import okhttp3.OkHttpClient;
58  import org.apache.commons.collections.CollectionUtils;
59  import org.apache.commons.io.IOUtils;
60  import org.apache.commons.lang3.RegExUtils;
61  import org.apache.commons.lang3.StringUtils;
62  
63  import java.io.ByteArrayInputStream;
64  import java.io.ByteArrayOutputStream;
65  import java.io.IOException;
66  import java.io.InputStream;
67  import java.io.OutputStream;
68  import java.net.InetSocketAddress;
69  import java.net.Proxy;
70  import java.net.URI;
71  import java.net.URISyntaxException;
72  import java.security.InvalidKeyException;
73  import java.security.NoSuchAlgorithmException;
74  import java.util.List;
75  
76  public class StockageService
77  {
78      private MinioClient _s3Client;
79  
80      // Properties
81      private final String _s3Url;
82      private final String _s3Bucket;
83      private final String _s3Key;
84      private final String _s3Password;
85  
86      private static final String SLASH = "/";
87      private static final String DOUBLE_SLASH = "//";
88  
89      private static final long DEFAULT_PART_SIZE = AppPropertiesService.getPropertyLong( "strois.default.partSize", 10485760L );
90  
91      public StockageService( String s3Url, String s3Bucket, String s3Key, String s3Password )
92      {
93          _s3Url = s3Url;
94          _s3Bucket = s3Bucket;
95          _s3Key = s3Key;
96          _s3Password = s3Password;
97      }
98  
99      /**
100      * Get S3 client to interact with NetApp server.
101      * 
102      * @return _s3Client
103      */
104     private MinioClient getS3Client( ) throws URISyntaxException
105     {
106         if ( _s3Client == null )
107         {
108             OkHttpClient okHttpClient = new OkHttpClient.Builder( ).proxy( getHttpAccessProxy( _s3Url ) ).build( );
109 
110             _s3Client = MinioClient.builder( ).endpoint( _s3Url ).credentials( _s3Key, _s3Password ).httpClient( okHttpClient ).build( );
111         }
112 
113         return _s3Client;
114     }
115 
116     /**
117      * Get proxy from httpaccess conf
118      * @param s3Url url to match to "noProxyFor"
119      * @return Proxy or Proxy.NO_PROXY
120      * @throws URISyntaxException
121      */
122     private Proxy getHttpAccessProxy( String s3Url ) throws URISyntaxException
123     {
124         String strProxyHost = AppPropertiesService.getProperty( "httpAccess.proxyHost" );
125         int proxyPort = AppPropertiesService.getPropertyInt( "httpAccess.proxyPort", 8080 );
126         String strNoProxyFor = AppPropertiesService.getProperty( "httpAccess.noProxyFor" );
127 
128         if ( StringUtils.isEmpty( strProxyHost ) )
129         {
130             return Proxy.NO_PROXY;
131         }
132 
133         boolean bNoProxy = StringUtils.isNotBlank( strNoProxyFor ) && S3Util.matchesList( strNoProxyFor.split( "," ), new URI( s3Url ).getHost( ) );
134 
135         if ( !bNoProxy )
136         {
137             InetSocketAddress proxyAddr = new InetSocketAddress( strProxyHost, proxyPort );
138             return new Proxy( Proxy.Type.HTTP, proxyAddr );
139         }
140         return Proxy.NO_PROXY;
141     }
142 
143     /**
144      * Load file from NetApp server
145      *
146      * @param pathToFile path to find file
147      * @return byte[] found
148      */
149     public byte[] loadFileFromNetAppServeur( String pathToFile ) throws MinioException
150     {
151         String completePathToFile = normalizeS3Path( pathToFile );
152         try ( InputStream is = getS3Client( ).getObject( GetObjectArgs.builder( ).bucket( _s3Bucket ).object( completePathToFile ).build( ) ) )
153         {
154             ByteArrayOutputStream output = new ByteArrayOutputStream( );
155             IOUtils.copy( is, output );
156             return output.toByteArray( );
157         }
158         catch( InvalidKeyException | IOException | NoSuchAlgorithmException | URISyntaxException e )
159         {
160             AppLogService.error( "Erreur chargement du fichier " + pathToFile, e );
161             throw new MinioException( "Erreur chargement du fichier " + pathToFile );
162         }
163         catch ( ErrorResponseException e )
164         {
165             AppLogService.error( "Erreur chargement du fichier " + pathToFile, e );
166             logErrorResponse( e );
167             throw new MinioException( "Erreur chargement du fichier " + pathToFile );
168         }
169         catch ( MinioException e )
170         {
171             AppLogService.error( "Erreur chargement du fichier " + pathToFile, e );
172             logMinioException( e );
173             throw new MinioException( "Erreur chargement du fichier " + pathToFile );
174         }
175     }
176 
177     /**
178      * Proceed save file.
179      *
180      * @param fileToSave
181      *            file content as byte[]
182      * @param pathToFile
183      *            path + filename to put file content in
184      *
185      * @return path to the photo on NetApp serveur
186      * @throws MinioException error uploading the file
187      *
188      */
189     public String saveFileToNetAppServer( byte [ ] fileToSave, String pathToFile ) throws MinioException
190     {
191         return saveFileToNetAppServer( new ByteArrayInputStream( fileToSave ), fileToSave.length, pathToFile, null );
192     }
193 
194     /**
195      * Proceed save file.
196      *
197      * @param fileToSave
198      *            file content as byte[]
199      * @param pathToFile
200      *            path + filename to put file content in
201      * @param userMetadata
202      *            user metadata
203      *
204      * @return path to the photo on NetApp serveur
205      *
206      */
207     public String saveFileToNetAppServer( byte [ ] fileToSave, String pathToFile, Multimap<String, String> userMetadata ) throws MinioException
208     {
209         return saveFileToNetAppServer( new ByteArrayInputStream( fileToSave ), fileToSave.length, pathToFile, userMetadata );
210     }
211 
212     /**
213      * Proceed save file.
214      *
215      * @param fileToSave
216      *            file content InputStream
217      * @param fileLength
218      *            file length (-1 if unknown, uses default part size then)
219      * @param pathToFile
220      *            path + filename to put file content in
221      *
222      * @return path to the photo on NetApp serveur
223      * @throws MinioException error uploading the file
224      *
225      */
226     public String saveFileToNetAppServer( InputStream fileToSave, long fileLength, String pathToFile ) throws MinioException
227     {
228         return saveFileToNetAppServer( fileToSave, fileLength, pathToFile, null );
229     }
230 
231     /**
232      * Proceed save file.
233      *
234      * @param fileToSave
235      *            file content InputStream
236      * @param fileLength
237      *            file length (-1 if unknown)
238      * @param pathToFile
239      *            path + filename to put file content in
240      * @param userMetadata
241      *            user metadata
242      * @return path to the photo on NetApp serveur
243      * @throws MinioException error uploading the file
244      *
245      */
246     public String saveFileToNetAppServer( InputStream fileToSave, long fileLength, String pathToFile, Multimap<String, String> userMetadata ) throws MinioException
247     {
248         if ( fileToSave == null || StringUtils.isEmpty( pathToFile ) )
249         {
250             return null;
251         }
252 
253         String completePathToFile = normalizeS3Path( pathToFile );
254         try
255         {
256             long partSize = -1;
257             if ( fileLength == -1 )
258             {
259                 partSize = DEFAULT_PART_SIZE;
260             }
261 
262             getS3Client( )
263                     .putObject( PutObjectArgs.builder( )
264                                         .bucket( _s3Bucket )
265                                         .object( completePathToFile )
266                                         .stream( fileToSave , fileLength, partSize )
267                                         .userMetadata( userMetadata )
268                                         .build( ) );
269         }
270         catch( InvalidKeyException | IOException | NoSuchAlgorithmException | URISyntaxException e )
271         {
272             AppLogService.error( "Erreur de sauvegarde du fichier " + completePathToFile, e );
273             throw new MinioException( "Erreur de sauvegarde du fichier " + completePathToFile );
274         }
275         catch ( ErrorResponseException e )
276         {
277             AppLogService.error( "Erreur de sauvegarde du fichier " + completePathToFile, e );
278             logErrorResponse( e );
279             throw new MinioException( "Erreur de sauvegarde du fichier " + completePathToFile );
280         }
281         catch ( MinioException e )
282         {
283             AppLogService.error( "Erreur de sauvegarde du fichier " + completePathToFile, e );
284             logMinioException( e );
285             throw new MinioException( "Erreur de sauvegarde du fichier " + completePathToFile );
286         }
287 
288         return completePathToFile;
289     }
290 
291     /**
292      * Proceed save files as single tarball archive in S3 root directory.
293      * @param objects list of files to upload as {@link SnowballObject}
294      * @return
295      * @throws MinioException error uploading the file(s)
296      */
297     public ObjectWriteResponse saveFileToNetAppServer( List<SnowballObject> objects ) throws MinioException
298     {
299         return saveFileToNetAppServer( objects, null );
300     }
301 
302     public ObjectWriteResponse saveFileToNetAppServer( List<SnowballObject> objects, Multimap<String, String> userMetadata ) throws MinioException
303     {
304         if ( CollectionUtils.isEmpty( objects ) )
305         {
306             return null;
307         }
308 
309         try
310         {
311             return getS3Client().uploadSnowballObjects(
312                     UploadSnowballObjectsArgs
313                             .builder( )
314                             .bucket( _s3Bucket )
315                             .objects(objects)
316                             .userMetadata( userMetadata )
317                             .build());
318         }
319         catch( InvalidKeyException | IOException | NoSuchAlgorithmException | URISyntaxException e )
320         {
321             AppLogService.error( "Erreur de sauvegarde du fichier", e );
322             throw new MinioException( "Erreur de sauvegarde du fichier " );
323         }
324         catch ( ErrorResponseException e )
325         {
326             AppLogService.error( "Erreur de sauvegarde du fichier ", e );
327             logErrorResponse( e );
328             throw new MinioException( "Erreur de sauvegarde du fichier " );
329         }
330         catch ( MinioException e )
331         {
332             AppLogService.error( "Erreur de sauvegarde du fichier ", e );
333             logMinioException( e );
334             throw new MinioException( "Erreur de sauvegarde du fichier " );
335         }
336     }
337 
338     /**
339      * Delete file on NetApp server.
340      * 
341      * @param pathToFile
342      *            file to delete, complete with file name
343      * @return false if error
344      */
345     public boolean deleteFileOnNetAppServeur( String pathToFile )
346     {
347 
348         if ( StringUtils.isEmpty( pathToFile ) )
349         {
350             AppLogService.debug( "Cannot delete file, pathToFile null or empty" );
351             return false;
352         }
353 
354         boolean result = true;
355 
356         String completePathToFile = normalizeS3Path( pathToFile );
357         AppLogService.debug( "File to delete " + completePathToFile );
358         try
359         {
360             getS3Client( ).removeObject( RemoveObjectArgs.builder( ).bucket( _s3Bucket ).object( completePathToFile ).build( ) );
361         }
362         catch( InvalidKeyException | IOException | NoSuchAlgorithmException | URISyntaxException e )
363         {
364             result = false;
365             AppLogService.error( "Erreur à la supression du fichier " + completePathToFile + " sur NetApp", e );
366         }
367         catch ( ErrorResponseException e )
368         {
369             logErrorResponse( e );
370             result = false;
371             AppLogService.error( "Erreur à la supression du fichier " + completePathToFile + " sur NetApp", e );
372         }
373         catch ( MinioException e )
374         {
375             logMinioException( e );
376             result = false;
377             AppLogService.error( "Erreur à la supression du fichier " + completePathToFile + " sur NetApp", e );
378         }
379 
380         AppLogService.debug( "Deleting file " + completePathToFile + " is " + ( result ? "OK" : "KO" ) );
381         return result;
382     }
383 
384     /**
385      *
386      * @param pathToFile path + filename
387      * @return Tags of the object
388      * @throws MinioException error getting the tags
389      */
390     public Tags getObjectTags( String pathToFile ) throws MinioException
391     {
392         try
393         {
394             return getS3Client( ).getObjectTags( GetObjectTagsArgs.builder( ).bucket( _s3Bucket ).object( pathToFile ).build( ) );
395         }
396         catch ( ErrorResponseException e )
397         {
398             AppLogService.error( "Erreur de récupération des tags du fichier ", e );
399             logErrorResponse( e );
400             throw new MinioException( "Erreur de récupération des tags du fichier " );
401         }
402         catch ( XmlParserException | ServerException | InvalidResponseException |
403                 InternalException | InsufficientDataException e )
404         {
405             AppLogService.error( "Erreur de récupération des tags du fichier " + pathToFile, e );
406             logMinioException( e );
407             throw new MinioException( "Erreur de récupération des tags du fichier " + pathToFile );
408         }
409         catch ( InvalidKeyException | URISyntaxException | NoSuchAlgorithmException | IOException e )
410         {
411             AppLogService.error( "Erreur de récupération des tags du fichier " + pathToFile, e );
412             throw new MinioException( "Erreur de récupération des tags du fichier " + pathToFile );
413         }
414     }
415 
416     /**
417      * Enables HTTP call tracing and written to traceStream.
418      *
419      * @param traceStream {@link OutputStream} for writing HTTP call tracing.
420      * @see #traceOff
421      */
422     public void traceOn( OutputStream traceStream ) throws URISyntaxException
423     {
424         getS3Client( ).traceOn( traceStream );
425     }
426 
427     /**
428      * Disables HTTP call tracing previously enabled.
429      *
430      * @see #traceOn
431      * @throws IOException upon connection error
432      */
433     public void traceOff( ) throws IOException, URISyntaxException
434     {
435         getS3Client( ).traceOff( );
436     }
437 
438     /**
439      * Sets HTTP connect, write and read timeouts. A value of 0 means no timeout, otherwise values
440      * must be between 1 and Integer.MAX_VALUE when converted to milliseconds.
441      *
442      * <pre>Example:{@code
443      * setTimeout(TimeUnit.SECONDS.toMillis(10), TimeUnit.SECONDS.toMillis(10),
444      *     TimeUnit.SECONDS.toMillis(30));
445      * }</pre>
446      *
447      * @param connectTimeout HTTP connect timeout in milliseconds.
448      * @param writeTimeout HTTP write timeout in milliseconds.
449      * @param readTimeout HTTP read timeout in milliseconds.
450      */
451     public void setTimeout(long connectTimeout, long writeTimeout, long readTimeout) throws URISyntaxException
452     {
453         getS3Client( ).setTimeout( connectTimeout, writeTimeout, readTimeout );
454     }
455 
456     private void logErrorResponse( ErrorResponseException e )
457     {
458         if ( e.errorResponse( ) != null )
459         {
460             AppLogService.debug( "errorResponse \n" + e.errorResponse( ) );
461         }
462         if ( e.httpTrace( ) != null )
463         {
464             AppLogService.debug( "httpTrace \n" + e.httpTrace( ) );
465         }
466         if ( e.getCause( ) != null )
467         {
468             AppLogService.debug( "Cause \n" + e.getCause( ) );
469         }
470     }
471     private void logMinioException( MinioException e )
472     {
473         if ( e.httpTrace( ) != null )
474         {
475             AppLogService.debug( "httpTrace \n" + e.httpTrace( ) );
476         }
477         if ( e.getCause( ) != null )
478         {
479             AppLogService.debug( "Cause \n" + e.getCause( ) );
480         }
481     }
482 
483     /**
484      * Replace "//" with "/" and delete leading "/" if exists
485      * @param path
486      * @return
487      */
488     private String normalizeS3Path( String path )
489     {
490         path = RegExUtils.replaceAll( path, DOUBLE_SLASH, SLASH );
491         path = StringUtils.removeStart( path, SLASH );
492         return path;
493     }
494 
495     @Override
496     public String toString( )
497     {
498         return "StockageService{" +
499                        "s3Url='" + _s3Url + '\'' +
500                        ", s3Bucket='" + _s3Bucket + '\'' +
501                        ", s3Key='" + StringUtils.abbreviate( _s3Key, 7 ) + '\'' +
502                        '}';
503     }
504 }