1 /*
2 * Copyright (c) 2002-2021, City of Paris
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 *
9 * 1. Redistributions of source code must retain the above copyright notice
10 * and the following disclaimer.
11 *
12 * 2. Redistributions in binary form must reproduce the above copyright notice
13 * and the following disclaimer in the documentation and/or other materials
14 * provided with the distribution.
15 *
16 * 3. Neither the name of 'Mairie de Paris' nor 'Lutece' nor the names of its
17 * contributors may be used to endorse or promote products derived from
18 * this software without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
24 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 * POSSIBILITY OF SUCH DAMAGE.
31 *
32 * License 1.0
33 */
34 package fr.paris.lutece.plugins.importexport.service.importdata.csvimportsource;
35
36 import fr.paris.lutece.plugins.importexport.business.ImportExportElement;
37 import fr.paris.lutece.plugins.importexport.service.importdata.IImportSource;
38 import fr.paris.lutece.portal.service.util.AppLogService;
39 import fr.paris.lutece.util.string.StringUtil;
40
41 import java.io.File;
42 import java.io.FileNotFoundException;
43 import java.io.FileReader;
44 import java.io.IOException;
45 import java.io.InputStreamReader;
46 import java.io.Reader;
47 import java.util.ArrayList;
48 import java.util.List;
49
50 import org.apache.commons.fileupload.FileItem;
51 import org.apache.commons.lang3.StringUtils;
52
53 import au.com.bytecode.opencsv.CSVReader;
54 import au.com.bytecode.opencsv.CSVParser;
55
56 /**
57 * Source to import data from CSV files.
58 */
59 public class CSVImportSource implements IImportSource
60 {
61 private static final char CONSTANT_BOM_UTF8 = 65279;
62
63 private CSVReader _csvReader;
64 private Reader _reader;
65 private List<String> _listColumnsName;
66
67 /**
68 * Creates a new CSVImportSource from a file uploaded by the user
69 *
70 * @param fileItem
71 * The CSV file to read data from. The first line of the CSV file must contain the names of columns of the database
72 * @param strCSVSeparator
73 * The CSV separator to use. If it is null or empty, then the default CSV separator is used (which is ",")
74 * @param strCSVQuoteChar
75 * The CSV quote character to use. If it is null or empty, then the default CSV quote character is used (which is "\"")
76 */
77 public CSVImportSource( FileItem fileItem, String strCSVSeparator, String strCSVQuoteChar )
78 {
79 if ( fileItem != null )
80 {
81
82 try
83 {
84 _reader = new InputStreamReader( fileItem.getInputStream( ) );
85 }
86 catch( IOException e )
87 {
88 AppLogService.error( e.getMessage( ), e );
89 }
90 if ( _reader != null )
91 {
92 Character cSeparator = StringUtils.isNotEmpty( strCSVSeparator ) ? strCSVSeparator.charAt( 0 ) : CSVParser.DEFAULT_SEPARATOR;
93 Character cQuoteChar = StringUtils.isNotEmpty( strCSVQuoteChar ) ? strCSVQuoteChar.charAt( 0 ) : CSVParser.DEFAULT_QUOTE_CHARACTER;
94 _csvReader = new CSVReader( _reader, cSeparator, cQuoteChar );
95 }
96 }
97 }
98
99 /**
100 * Creates a new CSVImportSource from a file of the server
101 *
102 * @param file
103 * The CSV file to read data from. The first line of the CSV file must contain the names of columns of the database
104 * @param strCSVSeparator
105 * The CSV separator to use. If it is null or empty, then the default CSV separator is used (which is ",")
106 * @param strCSVQuoteChar
107 * The CSV quote character to use. If it is null or empty, then the default CSV quote character is used (which is "\"")
108 */
109 public CSVImportSource( File file, String strCSVSeparator, String strCSVQuoteChar )
110 {
111 if ( file != null )
112 {
113 try
114 {
115 _reader = new FileReader( file );
116 Character cSeparator = StringUtils.isNotEmpty( strCSVSeparator ) ? strCSVSeparator.charAt( 0 ) : CSVParser.DEFAULT_SEPARATOR;
117 Character cQuoteChar = StringUtils.isNotEmpty( strCSVQuoteChar ) ? strCSVQuoteChar.charAt( 0 ) : CSVParser.DEFAULT_QUOTE_CHARACTER;
118 _csvReader = new CSVReader( _reader, cSeparator, cQuoteChar );
119 }
120 catch( FileNotFoundException e )
121 {
122 AppLogService.error( e.getMessage( ), e );
123 }
124 }
125 }
126
127 /**
128 * {@inheritDoc}
129 */
130 @Override
131 public List<ImportExportElement> getNextValues( )
132 {
133 // We check that the reader has been initialized
134 if ( _csvReader == null )
135 {
136 return null;
137 }
138 List<String> listColumnsName = getColumnsName( );
139 if ( listColumnsName == null )
140 {
141 return null;
142 }
143 String [ ] strLine;
144 try
145 {
146 strLine = _csvReader.readNext( );
147 }
148 catch( IOException e )
149 {
150 AppLogService.error( e.getMessage( ), e );
151 return null;
152 }
153 if ( strLine != null )
154 {
155 List<ImportExportElement> listElements = new ArrayList<ImportExportElement>( strLine.length );
156 if ( strLine.length != listColumnsName.size( ) )
157 {
158 // If the number of elements is not correct, then we return an empty list.
159 return listElements;
160 }
161 int i = 0;
162 for ( String strColumnTitle : listColumnsName )
163 {
164 ImportExportElementexport/business/ImportExportElement.html#ImportExportElement">ImportExportElement element = new ImportExportElement( );
165 element.setColumnName( strColumnTitle );
166 element.setValue( strLine [i] );
167 listElements.add( element );
168 i++;
169 }
170 return listElements;
171 }
172 return null;
173 }
174
175 /**
176 * {@inheritDoc}
177 */
178 @Override
179 public List<String> getColumnsName( )
180 {
181 if ( _listColumnsName != null )
182 {
183 return _listColumnsName;
184 }
185
186 if ( _csvReader == null )
187 {
188 return null;
189 }
190
191 String [ ] strFirstLine = null;
192 try
193 {
194 strFirstLine = _csvReader.readNext( );
195 }
196 catch( IOException e )
197 {
198 AppLogService.error( e.getMessage( ), e );
199 return null;
200 }
201
202 _listColumnsName = new ArrayList<String>( strFirstLine.length );
203 if ( strFirstLine.length > 0 )
204 {
205 boolean bIsFirst = true;
206 for ( String strColumnTitle : strFirstLine )
207 {
208 if ( strColumnTitle != null )
209 {
210 String strTitle = strColumnTitle.toLowerCase( ).trim( );
211 strTitle = StringUtil.replaceAccent( strTitle );
212 if ( bIsFirst )
213 {
214 // We check that the BOM character has not been read
215 if ( strTitle.startsWith( String.valueOf( CONSTANT_BOM_UTF8 ) ) )
216 {
217 strTitle = strTitle.substring( 1 );
218 }
219 // We eventually remove any first character that is not a latin character
220 else
221 if ( strTitle.charAt( 0 ) > 255 )
222 {
223 strTitle = strTitle.substring( 1 );
224 }
225
226 // The BOM character can only be the first character of the file
227 bIsFirst = false;
228 }
229 _listColumnsName.add( strTitle );
230 }
231 }
232 }
233
234 return _listColumnsName;
235 }
236
237 /**
238 * {@inheritDoc}
239 */
240 @Override
241 public void close( )
242 {
243 if ( _csvReader != null )
244 {
245 try
246 {
247 _csvReader.close( );
248 _csvReader = null;
249 _reader.close( );
250 }
251 catch( IOException e )
252 {
253 AppLogService.error( e.getMessage( ), e );
254 }
255 }
256 }
257
258 /**
259 * {@inheritDoc}
260 */
261 @Override
262 protected void finalize( ) throws Throwable
263 {
264 if ( _csvReader != null )
265 {
266 close( );
267 }
268 super.finalize( );
269 }
270 }