View Javadoc
1   /*
2    * Copyright (c) 2002-2020, City of Paris
3    * All rights reserved.
4    *
5    * Redistribution and use in source and binary forms, with or without
6    * modification, are permitted provided that the following conditions
7    * are met:
8    *
9    *  1. Redistributions of source code must retain the above copyright notice
10   *     and the following disclaimer.
11   *
12   *  2. Redistributions in binary form must reproduce the above copyright notice
13   *     and the following disclaimer in the documentation and/or other materials
14   *     provided with the distribution.
15   *
16   *  3. Neither the name of 'Mairie de Paris' nor 'Lutece' nor the names of its
17   *     contributors may be used to endorse or promote products derived from
18   *     this software without specific prior written permission.
19   *
20   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21   * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22   * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23   * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE
24   * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25   * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26   * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27   * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28   * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30   * POSSIBILITY OF SUCH DAMAGE.
31   *
32   * License 1.0
33   */
34  package fr.paris.lutece.plugins.announce.service.announcesearch;
35  
36  import java.text.ParseException;
37  import java.text.SimpleDateFormat;
38  import java.util.ArrayList;
39  import java.util.Collection;
40  import java.util.Date;
41  import java.util.List;
42  import java.util.Locale;
43  
44  import org.apache.commons.lang.StringUtils;
45  import org.apache.lucene.document.DateTools;
46  import org.apache.lucene.document.Document;
47  import org.apache.lucene.index.Term;
48  import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
49  import org.apache.lucene.search.BooleanClause;
50  import org.apache.lucene.search.IndexSearcher;
51  import org.apache.lucene.search.PhraseQuery;
52  import org.apache.lucene.search.Query;
53  import org.apache.lucene.search.ScoreDoc;
54  import org.apache.lucene.search.TermQuery;
55  import org.apache.lucene.search.TermRangeQuery;
56  import org.apache.lucene.search.TopDocs;
57  import org.apache.lucene.util.BytesRef;
58  
59  import fr.paris.lutece.plugins.announce.business.Announce;
60  import fr.paris.lutece.plugins.announce.business.AnnounceHome;
61  import fr.paris.lutece.plugins.announce.business.AnnounceSearchFilter;
62  import fr.paris.lutece.plugins.announce.business.AnnounceSort;
63  import fr.paris.lutece.plugins.announce.service.AnnouncePlugin;
64  import fr.paris.lutece.portal.service.plugin.Plugin;
65  import fr.paris.lutece.portal.service.search.SearchItem;
66  import fr.paris.lutece.portal.service.search.SearchResult;
67  import fr.paris.lutece.portal.service.util.AppLogService;
68  
69  /**
70   * LuceneSearchEngine
71   */
72  public class AnnounceLuceneSearchEngine implements IAnnounceSearchEngine
73  {
74      private static final int NO_CATEGORY = 0;
75      private static final int NO_SECTOR = 0;
76      private final SimpleDateFormat _dayFormat = new SimpleDateFormat( "yyyyMMdd", Locale.US );
77  
78      /**
79       * {@inheritDoc}
80       */
81      @Override
82      public int getSearchResults( AnnounceSearchFilter filter, Plugin plugin, List<SearchResult> listSearchResult, int nPage, int nItemsPerPage )
83      {
84          ArrayList<SearchItem> listResults = new ArrayList<>( );
85          IndexSearcher searcher;
86  
87          int nNbResults = 0;
88  
89          try
90          {
91              searcher = AnnounceSearchService.getInstance( ).getSearcher( );
92  
93              Collection<String> queries = new ArrayList<>( );
94              Collection<String> sectors = new ArrayList<>( );
95              Collection<BooleanClause.Occur> flags = new ArrayList<>( );
96  
97              // Category id
98              if ( filter.getIdCategory( ) != NO_CATEGORY )
99              {
100                 Query queryCategoryId = new TermQuery( new Term( AnnounceSearchItem.FIELD_CATEGORY_ID, String.valueOf( filter.getIdCategory( ) ) ) );
101                 queries.add( queryCategoryId.toString( ) );
102                 sectors.add( AnnounceSearchItem.FIELD_CATEGORY_ID );
103                 flags.add( BooleanClause.Occur.MUST );
104             }
105 
106             // Category id
107             if ( filter.getIdSector( ) != NO_SECTOR )
108             {
109                 Query querySectorId = new TermQuery( new Term( AnnounceSearchItem.FIELD_SECTOR_ID, String.valueOf( filter.getIdSector( ) ) ) );
110                 queries.add( querySectorId.toString( ) );
111                 sectors.add( AnnounceSearchItem.FIELD_SECTOR_ID );
112                 flags.add( BooleanClause.Occur.MUST );
113             }
114 
115             // Type (=announce)
116             PhraseQuery.Builder queryTypeBuilder = new PhraseQuery.Builder( );
117             // add character "e" to TYPE because field is not analyzed when added to lucene document, but it's analyzed then in MultiFieldQueryParser.parse
118             // method
119             queryTypeBuilder.add( new Term( SearchItem.FIELD_TYPE, AnnouncePlugin.PLUGIN_NAME + "e" ) );
120             queries.add( queryTypeBuilder.build( ).toString( ) );
121             sectors.add( SearchItem.FIELD_TYPE );
122             flags.add( BooleanClause.Occur.MUST );
123 
124             // Keywords in title or description
125             if ( StringUtils.isNotBlank( filter.getKeywords( ) ) )
126             {
127                 PhraseQuery.Builder queryContentBuilder = new PhraseQuery.Builder( );
128                 queryContentBuilder.add( new Term( SearchItem.FIELD_CONTENTS, filter.getKeywords( ) ) );
129                 queries.add( queryContentBuilder.build( ).toString( ) );
130                 sectors.add( SearchItem.FIELD_CONTENTS );
131                 flags.add( BooleanClause.Occur.MUST );
132             }
133 
134             // contains range date
135             if ( ( filter.getDateMin( ) != null ) || ( filter.getDateMax( ) != null ) )
136             {
137                 Date dateMinToSearch = new Date( 0L );
138                 Date dateMaxToSearch = new Date( );
139                 if ( filter.getDateMin( ) != null )
140                 {
141                     dateMinToSearch = filter.getDateMin( );
142                 }
143 
144                 if ( filter.getDateMax( ) != null )
145                 {
146                     dateMaxToSearch = filter.getDateMax( );
147                 }
148 
149                 // String stringDateMin = DateUtil.
150                 String strLowerTerm = _dayFormat.format( dateMinToSearch );
151                 String strUpperTerm = _dayFormat.format( dateMaxToSearch );
152                 BytesRef bRLowerTerm = new BytesRef( strLowerTerm );
153                 BytesRef bRUpperTerm = new BytesRef( strUpperTerm );
154                 Query queryRangeDate = new TermRangeQuery( SearchItem.FIELD_DATE, bRLowerTerm, bRUpperTerm, true, true );
155                 queries.add( queryRangeDate.toString( ) );
156                 sectors.add( SearchItem.FIELD_DATE );
157                 flags.add( BooleanClause.Occur.MUST );
158             }
159 
160             // contains range price
161             if ( ( filter.getPriceMin( ) > 0 ) || ( filter.getPriceMax( ) > 0 ) )
162             {
163                 int nPriceMin = ( filter.getPriceMin( ) > 0 ) ? filter.getPriceMin( ) : 0;
164                 int nPriceMax = ( filter.getPriceMax( ) > 0 ) ? filter.getPriceMax( ) : Integer.MAX_VALUE;
165                 Query queryRangePrice = new TermRangeQuery( AnnounceSearchItem.FIELD_PRICE,
166                         new BytesRef( AnnounceSearchService.formatPriceForIndexer( nPriceMin ) ),
167                         new BytesRef( AnnounceSearchService.formatPriceForIndexer( nPriceMax ) ), true, true );
168                 queries.add( queryRangePrice.toString( ) );
169                 sectors.add( AnnounceSearchItem.FIELD_PRICE );
170                 flags.add( BooleanClause.Occur.MUST );
171             }
172 
173             Query queryMulti = MultiFieldQueryParser.parse( queries.toArray( new String [ queries.size( )] ), sectors.toArray( new String [ sectors.size( )] ),
174                     flags.toArray( new BooleanClause.Occur [ flags.size( )] ), AnnounceSearchService.getInstance( ).getAnalyzer( ) );
175 
176             TopDocs topDocs = searcher.search( queryMulti, 1000000 );
177             ScoreDoc [ ] hits = topDocs.scoreDocs;
178             nNbResults = hits.length;
179 
180             // We only get the documents of the current page
181             int nFrom = ( nPage - 1 ) * nItemsPerPage;
182 
183             if ( nFrom < 0 )
184             {
185                 nFrom = 0;
186             }
187 
188             int nTo = ( nPage * nItemsPerPage );
189 
190             if ( ( nTo == 0 ) || ( nTo > nNbResults ) )
191             {
192                 nTo = nNbResults;
193             }
194 
195             for ( int i = nFrom; i < nTo; i++ )
196             {
197                 int docId = hits [i].doc;
198                 Document document = searcher.doc( docId );
199                 SearchItem si = new SearchItem( document );
200                 listResults.add( si );
201             }
202         }
203         catch( Exception e )
204         {
205             AppLogService.error( e.getMessage( ), e );
206         }
207         convertList( listResults, listSearchResult );
208 
209         return nNbResults;
210     }
211 
212     /**
213      * {@inheritDoc}
214      */
215     @Override
216     public int getSearchResultsBis( AnnounceSearchFilter filter, Plugin plugin, List<Announce> listAnnouncesResult, int nPage, int nItemsPerPage,
217             AnnounceSort anSort )
218     {
219         ArrayList<SearchItem> listResults = new ArrayList<>( );
220         List<Integer> listIdAnnounces = new ArrayList<>( );
221         IndexSearcher searcher;
222 
223         Date dateMinToSearch;
224         Date dateMaxToSearch;
225         int nNbResults = 0;
226         try
227         {
228             searcher = AnnounceSearchService.getInstance( ).getSearcher( );
229 
230             Collection<String> queries = new ArrayList<>( );
231             Collection<String> sectors = new ArrayList<>( );
232             Collection<BooleanClause.Occur> flags = new ArrayList<>( );
233 
234             // Category id
235             if ( filter.getIdCategory( ) != NO_CATEGORY )
236             {
237                 Query queryCategoryId = new TermQuery( new Term( AnnounceSearchItem.FIELD_CATEGORY_ID, String.valueOf( filter.getIdCategory( ) ) ) );
238                 queries.add( queryCategoryId.toString( ) );
239                 sectors.add( AnnounceSearchItem.FIELD_CATEGORY_ID );
240                 flags.add( BooleanClause.Occur.MUST );
241             }
242 
243             // Category id
244             if ( filter.getIdSector( ) != NO_SECTOR )
245             {
246                 Query querySectorId = new TermQuery( new Term( AnnounceSearchItem.FIELD_SECTOR_ID, String.valueOf( filter.getIdSector( ) ) ) );
247                 queries.add( querySectorId.toString( ) );
248                 sectors.add( AnnounceSearchItem.FIELD_SECTOR_ID );
249                 flags.add( BooleanClause.Occur.MUST );
250             }
251 
252             // Type (=announce)
253             PhraseQuery.Builder queryTypeBuilder = new PhraseQuery.Builder( );
254             // add character "e" to TYPE because field is not analyzed when added to lucene document, but it's analyzed then in MultiFieldQueryParser.parse
255             // method
256             queryTypeBuilder.add( new Term( SearchItem.FIELD_TYPE, AnnouncePlugin.PLUGIN_NAME + "e" ) );
257             queries.add( queryTypeBuilder.build( ).toString( ) );
258             sectors.add( SearchItem.FIELD_TYPE );
259             flags.add( BooleanClause.Occur.MUST );
260 
261             // Keywords in title or description
262             if ( StringUtils.isNotBlank( filter.getKeywords( ) ) )
263             {
264                 PhraseQuery.Builder queryContentBuilder = new PhraseQuery.Builder( );
265                 queryContentBuilder.add( new Term( SearchItem.FIELD_CONTENTS, filter.getKeywords( ) ) );
266                 queries.add( queryContentBuilder.build( ).toString( ) );
267                 sectors.add( SearchItem.FIELD_CONTENTS );
268                 flags.add( BooleanClause.Occur.MUST );
269             }
270 
271             // contains range date
272             if ( ( filter.getDateMin( ) != null ) || ( filter.getDateMax( ) != null ) )
273             {
274                 if ( filter.getDateMin( ) == null )
275                 {
276                     dateMinToSearch = new Date( 0L );
277                 }
278                 else
279                 {
280                     dateMinToSearch = filter.getDateMin( );
281                 }
282 
283                 if ( filter.getDateMax( ) == null )
284                 {
285                     dateMaxToSearch = new Date( );
286                 }
287                 else
288                 {
289                     dateMaxToSearch = filter.getDateMax( );
290                 }
291 
292                 // String stringDateMin = DateUtil.
293                 String strLowerTerm = _dayFormat.format( dateMinToSearch );
294                 String strUpperTerm = _dayFormat.format( dateMaxToSearch );
295                 BytesRef bRLowerTerm = new BytesRef( strLowerTerm );
296                 BytesRef bRUpperTerm = new BytesRef( strUpperTerm );
297                 Query queryRangeDate = new TermRangeQuery( SearchItem.FIELD_DATE, bRLowerTerm, bRUpperTerm, true, true );
298                 queries.add( queryRangeDate.toString( ) );
299                 sectors.add( SearchItem.FIELD_DATE );
300                 flags.add( BooleanClause.Occur.MUST );
301             }
302 
303             // contains range price
304             if ( ( filter.getPriceMin( ) > 0 ) || ( filter.getPriceMax( ) > 0 ) )
305             {
306                 int nPriceMin = ( filter.getPriceMin( ) > 0 ) ? filter.getPriceMin( ) : 0;
307                 int nPriceMax = ( filter.getPriceMax( ) > 0 ) ? filter.getPriceMax( ) : Integer.MAX_VALUE;
308                 Query queryRangePrice = new TermRangeQuery( AnnounceSearchItem.FIELD_PRICE,
309                         new BytesRef( AnnounceSearchService.formatPriceForIndexer( nPriceMin ) ),
310                         new BytesRef( AnnounceSearchService.formatPriceForIndexer( nPriceMax ) ), true, true );
311                 queries.add( queryRangePrice.toString( ) );
312                 sectors.add( AnnounceSearchItem.FIELD_PRICE );
313                 flags.add( BooleanClause.Occur.MUST );
314             }
315 
316             Query queryMulti = MultiFieldQueryParser.parse( queries.toArray( new String [ queries.size( )] ), sectors.toArray( new String [ sectors.size( )] ),
317                     flags.toArray( new BooleanClause.Occur [ flags.size( )] ), AnnounceSearchService.getInstance( ).getAnalyzer( ) );
318 
319             TopDocs topDocs = searcher.search( queryMulti, 1000000 );
320             ScoreDoc [ ] hits = topDocs.scoreDocs;
321             nNbResults = hits.length;
322 
323             // -------------------------------------------------
324             for ( int i = 0; i < nNbResults; i++ )
325             {
326                 int docId = hits [i].doc;
327                 Document document = searcher.doc( docId );
328                 SearchItem si = new SearchItem( document );
329                 listResults.add( si );
330             }
331             for ( SearchItem searchResult : listResults )
332             {
333                 if ( searchResult.getId( ) != null )
334                 {
335                     listIdAnnounces.add( Integer.parseInt( searchResult.getId( ) ) );
336                 }
337             }
338 
339             List<Announce> listAnnounces = AnnounceHome.findByListId( listIdAnnounces, anSort );
340 
341             // -------------------------------------------------
342 
343             // We only get the documents of the current page
344             int nFrom = ( nPage - 1 ) * nItemsPerPage;
345 
346             if ( nFrom < 0 )
347             {
348                 nFrom = 0;
349             }
350 
351             int nTo = ( nPage * nItemsPerPage );
352 
353             if ( ( nTo == 0 ) || ( nTo > nNbResults ) )
354             {
355                 nTo = nNbResults;
356             }
357 
358             for ( int i = nFrom; i < nTo; i++ )
359             {
360                 listAnnouncesResult.add( listAnnounces.get( i ) );
361             }
362         }
363         catch( Exception e )
364         {
365             AppLogService.error( e.getMessage( ), e );
366         }
367 
368         return nNbResults;
369     }
370 
371     /**
372      * Convert the SearchItem list on SearchResult list
373      * 
374      * @param listSource
375      *            The source list
376      * @param listSearchResult
377      *            The result list
378      */
379     private void convertList( List<SearchItem> listSource, List<SearchResult> listSearchResult )
380     {
381         for ( SearchItem item : listSource )
382         {
383             SearchResult result = new SearchResult( );
384             result.setId( item.getId( ) );
385 
386             try
387             {
388                 result.setDate( DateTools.stringToDate( item.getDate( ) ) );
389             }
390             catch( ParseException e )
391             {
392                 AppLogService.error( "Bad Date Format for indexed item \"" + item.getTitle( ) + "\" : " + e.getMessage( ) );
393             }
394 
395             result.setUrl( item.getUrl( ) );
396             result.setTitle( item.getTitle( ) );
397             result.setSummary( item.getSummary( ) );
398             result.setType( item.getType( ) );
399             listSearchResult.add( result );
400         }
401     }
402 }