View Javadoc
1   /*
2    * Copyright (c) 2002-2022, City of Paris
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without
6    * modification, are permitted provided that the following conditions
7    * are met:
8    *
9    *  1. Redistributions of source code must retain the above copyright notice
10   *     and the following disclaimer.
11   *
12   *  2. Redistributions in binary form must reproduce the above copyright notice
13   *     and the following disclaimer in the documentation and/or other materials
14   *     provided with the distribution.
15   *
16   *  3. Neither the name of 'Mairie de Paris' nor 'Lutece' nor the names of its
17   *     contributors may be used to endorse or promote products derived from
18   *     this software without specific prior written permission.
19   *
20   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
24   * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   * POSSIBILITY OF SUCH DAMAGE.
31   *
32   * License 1.0
33   */
34  package fr.paris.lutece.portal.service.search;
35  
36  import java.text.ParseException;
37  import java.util.ArrayList;
38  import java.util.Arrays;
39  import java.util.Date;
40  import java.util.List;
41  import java.util.Locale;
42  
43  import javax.servlet.http.HttpServletRequest;
44  
45  import org.apache.commons.collections.CollectionUtils;
46  import org.apache.commons.lang3.StringUtils;
47  import org.apache.commons.lang3.ArrayUtils;
48  import org.apache.lucene.document.DateTools;
49  import org.apache.lucene.document.DateTools.Resolution;
50  import org.apache.lucene.document.Document;
51  import org.apache.lucene.index.DirectoryReader;
52  import org.apache.lucene.index.IndexReader;
53  import org.apache.lucene.index.Term;
54  import org.apache.lucene.queryparser.classic.QueryParser;
55  import org.apache.lucene.queryparser.classic.QueryParserBase;
56  import org.apache.lucene.search.BooleanClause;
57  import org.apache.lucene.search.BooleanQuery;
58  import org.apache.lucene.search.IndexSearcher;
59  import org.apache.lucene.search.Query;
60  import org.apache.lucene.search.ScoreDoc;
61  import org.apache.lucene.search.TermQuery;
62  import org.apache.lucene.search.TermRangeQuery;
63  import org.apache.lucene.search.TopDocs;
64  import org.apache.lucene.store.Directory;
65  import org.apache.lucene.util.BytesRef;
66  
67  import fr.paris.lutece.portal.business.page.Page;
68  import fr.paris.lutece.portal.service.security.LuteceUser;
69  import fr.paris.lutece.portal.service.security.SecurityService;
70  import fr.paris.lutece.portal.service.util.AppLogService;
71  import fr.paris.lutece.util.date.DateUtil;
72  
73  /**
74   * LuceneSearchEngine
75   */
76  public class LuceneSearchEngine implements SearchEngine
77  {
78      public static final int MAX_RESPONSES = 1000000;
79      private static final String PARAMETER_TYPE_FILTER = "type_filter";
80      private static final String PARAMETER_DATE_AFTER = "date_after";
81      private static final String PARAMETER_DATE_BEFORE = "date_before";
82      private static final String PARAMETER_TAG_FILTER = "tag_filter";
83      private static final String PARAMETER_DEFAULT_OPERATOR = "default_operator";
84      private static final String PARAMETER_OPERATOR_AND = "AND";
85  
86      /**
87       * Return search results
88       *
89       * @param strQuery
90       *            The search query
91       * @param request
92       *            The HTTP request
93       * @return Results as a collection of SearchResult
94       */
95      public List<SearchResult> getSearchResults( String strQuery, HttpServletRequest request )
96      {
97          List<Query> listFilter = new ArrayList<>( );
98          boolean bFilterResult = false;
99  
100         if ( SecurityService.isAuthenticationEnable( ) )
101         {
102             LuteceUser user = SecurityService.getInstance( ).getRegisteredUser( request );
103 
104             Query [ ] filtersRole = null;
105 
106             if ( user != null )
107             {
108                 String [ ] userRoles = SecurityService.getInstance( ).getRolesByUser( user );
109 
110                 if ( userRoles != null )
111                 {
112                     filtersRole = new Query [ userRoles.length + 1];
113 
114                     for ( int i = 0; i < userRoles.length; i++ )
115                     {
116                         Query queryRole = new TermQuery( new Term( SearchItem.FIELD_ROLE, userRoles [i] ) );
117                         filtersRole [i] = queryRole;
118                     }
119                 }
120                 else
121                 {
122                     bFilterResult = true;
123                 }
124             }
125             else
126             {
127                 filtersRole = new Query [ 1];
128             }
129 
130             if ( !bFilterResult )
131             {
132                 Query queryRole = new TermQuery( new Term( SearchItem.FIELD_ROLE, Page.ROLE_NONE ) );
133                 filtersRole [filtersRole.length - 1] = queryRole;
134                 BooleanQuery.Builder booleanQueryBuilderRole = new BooleanQuery.Builder( );
135                 Arrays.asList( filtersRole ).stream( ).forEach( filterRole -> booleanQueryBuilderRole.add( filterRole, BooleanClause.Occur.SHOULD ) );
136 
137                 listFilter.add( booleanQueryBuilderRole.build( ) );
138             }
139         }
140 
141         String [ ] typeFilter = request.getParameterValues( PARAMETER_TYPE_FILTER );
142         String strDateAfter = request.getParameter( PARAMETER_DATE_AFTER );
143         String strDateBefore = request.getParameter( PARAMETER_DATE_BEFORE );
144         Query allFilter = buildFinalFilter( listFilter, strDateAfter, strDateBefore, typeFilter, request.getLocale( ) );
145 
146         String strTagFilter = request.getParameter( PARAMETER_TAG_FILTER );
147         return search( strTagFilter, strQuery, allFilter, request, bFilterResult );
148     }
149 
150     private Query buildFinalFilter( List<Query> listFilter, String strDateAfter, String strDateBefore, String [ ] typeFilter, Locale locale )
151     {
152         Query filterDate = createFilterDate( strDateAfter, strDateBefore, locale );
153         if ( filterDate != null )
154         {
155             listFilter.add( filterDate );
156         }
157 
158         Query filterType = createFilterType( typeFilter );
159         if ( filterType != null )
160         {
161             listFilter.add( filterType );
162         }
163 
164         Query allFilter = null;
165         if ( CollectionUtils.isNotEmpty( listFilter ) )
166         {
167             BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder( );
168             for ( Query filter : listFilter )
169             {
170                 booleanQueryBuilder.add( filter, BooleanClause.Occur.MUST );
171             }
172             allFilter = booleanQueryBuilder.build( );
173         }
174         return allFilter;
175     }
176 
177     private Query createFilterType( String [ ] typeFilter )
178     {
179         if ( ArrayUtils.isNotEmpty( typeFilter ) && !typeFilter [0].equals( SearchService.TYPE_FILTER_NONE ) )
180         {
181             BooleanQuery.Builder booleanQueryBuilder = new BooleanQuery.Builder( );
182             for ( int i = 0; i < typeFilter.length; i++ )
183             {
184                 Query queryType = new TermQuery( new Term( SearchItem.FIELD_TYPE, typeFilter [i] ) );
185                 booleanQueryBuilder.add( queryType, BooleanClause.Occur.SHOULD );
186             }
187             return booleanQueryBuilder.build( );
188         }
189         return null;
190     }
191 
192     private Query createFilterDate( String strDateAfter, String strDateBefore, Locale locale )
193     {
194         boolean bDateAfter = false;
195         boolean bDateBefore = false;
196 
197         if ( StringUtils.isNotBlank( strDateAfter ) || StringUtils.isNotBlank( strDateBefore ) )
198         {
199             BytesRef strAfter = null;
200             BytesRef strBefore = null;
201 
202             if ( StringUtils.isNotBlank( strDateAfter ) )
203             {
204                 Date dateAfter = DateUtil.formatDate( strDateAfter, locale );
205                 strAfter = new BytesRef( DateTools.dateToString( dateAfter, Resolution.DAY ) );
206                 bDateAfter = true;
207             }
208 
209             if ( StringUtils.isNotBlank( strDateBefore ) )
210             {
211                 Date dateBefore = DateUtil.formatDate( strDateBefore, locale );
212                 strBefore = new BytesRef( DateTools.dateToString( dateBefore, Resolution.DAY ) );
213                 bDateBefore = true;
214             }
215 
216             return new TermRangeQuery( SearchItem.FIELD_DATE, strAfter, strBefore, bDateAfter, bDateBefore );
217         }
218         return null;
219     }
220 
221     private List<SearchResult> search( String strTagFilter, String strQuery, Query allFilter, HttpServletRequest request, boolean bFilterResult )
222     {
223         List<SearchItem> listResults = new ArrayList<>( );
224         try ( Directory directory = IndexationService.getDirectoryIndex( ) ; IndexReader ir = DirectoryReader.open( directory ) ; )
225         {
226             IndexSearcher searcher = new IndexSearcher( ir );
227 
228             BooleanQuery.Builder bQueryBuilder = new BooleanQuery.Builder( );
229 
230             if ( StringUtils.isNotBlank( strTagFilter ) )
231             {
232                 QueryParser parser = new QueryParser( SearchItem.FIELD_METADATA, IndexationService.getAnalyser( ) );
233 
234                 String formatQuery = ( strQuery != null ) ? strQuery : "";
235 
236                 Query queryMetaData = parser.parse( formatQuery );
237                 bQueryBuilder.add( queryMetaData, BooleanClause.Occur.SHOULD );
238 
239                 parser = new QueryParser( SearchItem.FIELD_SUMMARY, IndexationService.getAnalyser( ) );
240 
241                 Query querySummary = parser.parse( formatQuery );
242                 bQueryBuilder.add( querySummary, BooleanClause.Occur.SHOULD );
243             }
244             else
245             {
246                 QueryParser parser = new QueryParser( SearchItem.FIELD_CONTENTS, IndexationService.getAnalyser( ) );
247 
248                 String operator = request.getParameter( PARAMETER_DEFAULT_OPERATOR );
249 
250                 if ( PARAMETER_OPERATOR_AND.equals( operator ) )
251                 {
252                     parser.setDefaultOperator( QueryParserBase.AND_OPERATOR );
253                 }
254 
255                 Query queryContent = parser.parse( ( StringUtils.isNotBlank( strQuery ) ) ? strQuery : "" );
256                 bQueryBuilder.add( queryContent, BooleanClause.Occur.SHOULD );
257             }
258 
259             Query query = bQueryBuilder.build( );
260 
261             if ( allFilter != null )
262             {
263                 BooleanQuery.Builder bQueryBuilderWithFilter = new BooleanQuery.Builder( );
264                 bQueryBuilderWithFilter.add( allFilter, BooleanClause.Occur.FILTER );
265                 bQueryBuilderWithFilter.add( query, BooleanClause.Occur.MUST );
266                 query = bQueryBuilderWithFilter.build( );
267             }
268 
269             // Get results documents
270             TopDocs topDocs = searcher.search( query, MAX_RESPONSES );
271             ScoreDoc [ ] hits = topDocs.scoreDocs;
272 
273             for ( int i = 0; i < hits.length; i++ )
274             {
275                 int docId = hits [i].doc;
276                 Document document = searcher.doc( docId );
277                 SearchItem/service/search/SearchItem.html#SearchItem">SearchItem si = new SearchItem( document );
278 
279                 if ( ( !bFilterResult ) || ( si.getRole( ).equals( Page.ROLE_NONE ) )
280                         || ( SecurityService.getInstance( ).isUserInRole( request, si.getRole( ) ) ) )
281                 {
282                     listResults.add( si );
283                 }
284             }
285         }
286         catch( Exception e )
287         {
288             AppLogService.error( e.getMessage( ), e );
289         }
290         return convertList( listResults );
291     }
292 
293     /**
294      * Convert a list of Lucene items into a list of generic search items
295      * 
296      * @param listSource
297      *            The list of Lucene items
298      * @return A list of generic search items
299      */
300     private List<SearchResult> convertList( List<SearchItem> listSource )
301     {
302         List<SearchResult> listDest = new ArrayList<>( );
303 
304         for ( SearchItem item : listSource )
305         {
306             SearchResultce/search/SearchResult.html#SearchResult">SearchResult result = new SearchResult( );
307             result.setId( item.getId( ) );
308 
309             try
310             {
311                 result.setDate( DateTools.stringToDate( item.getDate( ) ) );
312             }
313             catch( ParseException e )
314             {
315                 AppLogService.error( "Bad Date Format for indexed item \"{}\" : {}", item.getTitle( ), e.getMessage( ), e );
316             }
317 
318             result.setUrl( item.getUrl( ) );
319             result.setTitle( item.getTitle( ) );
320             result.setSummary( item.getSummary( ) );
321             result.setType( item.getType( ) );
322             listDest.add( result );
323         }
324 
325         return listDest;
326     }
327 }