Lucene - Sorting

---

---

Lucene gives the search results by default sorted by relevance and which can be manipulated as required.

Sorting by Relevance is the default sorting mode used by Lucene. Lucene provides results by the most relevant hit at the top.

Steps to sort Search results

Step 1: Create Index for the item to be sorted.

Add SortedDocValuesField for the field to be sorted.

//index file name
Field fileNameField = new Field(LuceneConstants.FILE_NAME, file.getName(),type);
//sort file name
Field sortedFileNameField = new SortedDocValuesField(LuceneConstants.FILE_NAME, new BytesRef(file.getName()));

// add fields
document.add(fileNameField);
document.add(sortedFileNameField);

Step 2: Create SortField and Sort Objects

Create Sort Object for the field to be searched.

// Sort by a string field  
SortField fileNameSort = new SortField(LuceneConstants.FILE_NAME, SortField.Type.STRING); 
Sort sort = new Sort(fileNameSort);

Step 3: Search using Sort Object

// sort and return search results
return indexSearcher.search(query, LuceneConstants.MAX_SEARCH, sort);

Example Application

To test sorting by relevance, let us create a test Lucene application.

LuceneConstants.java

This class is used to provide various constants to be used across the sample application.

package com.tutorialspoint.lucene;

public class LuceneConstants {
   public static final String CONTENTS = "contents";
   public static final String FILE_NAME = "filename";
   public static final String FILE_PATH = "filepath";
   public static final int MAX_SEARCH = 10;
}

TextFileFilter.java

This class is used as a .txt file filter.

package com.tutorialspoint.lucene;

import java.io.File;
import java.io.FileFilter;

public class TextFileFilter implements FileFilter {

   @Override
   public boolean accept(File pathname) {
      return pathname.getName().toLowerCase().endsWith(".txt");
   }
}

Indexer.java

This class is used to create index using lucene library.

package com.tutorialspoint.lucene;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileReader;
import java.io.IOException;
import java.nio.file.Paths;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.IndexOptions;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;

public class Indexer {

   private IndexWriter writer;

   public Indexer(String indexDirectoryPath) throws IOException {
      //this directory will contain the indexes
	  Directory indexDirectory = FSDirectory.open(Paths.get(indexDirectoryPath));
      StandardAnalyzer analyzer = new StandardAnalyzer();
      IndexWriterConfig config = new IndexWriterConfig(analyzer);
      writer = new IndexWriter(indexDirectory, config);
   }

   public void close() throws CorruptIndexException, IOException {
      writer.close();
   }
   
   private Document getDocument(File file) throws IOException {
      Document document = new Document();

      //index file contents
      Field contentField = new TextField(LuceneConstants.CONTENTS, 
      new FileReader(file));

      FieldType type = new FieldType();
      type.setStored(true);
      type.setTokenized(false);
      type.setIndexOptions(IndexOptions.DOCS);
      type.setOmitNorms(true);

      //index file name
      Field fileNameField = new Field(LuceneConstants.FILE_NAME,
      file.getName(),type);

      //sort file name
      Field sortedFileNameField = new SortedDocValuesField(LuceneConstants.FILE_NAME, new BytesRef(file.getName()));

      //index file path
      Field filePathField = new Field(LuceneConstants.FILE_PATH,
      file.getCanonicalPath(),type);

      document.add(contentField);
      document.add(fileNameField);
      document.add(sortedFileNameField);
      document.add(filePathField);

      return document;
   }   

   private void indexFile(File file) throws IOException {
      System.out.println("Indexing "+file.getCanonicalPath());
      Document document = getDocument(file);
      writer.addDocument(document);
   }

   public int createIndex(String dataDirPath, FileFilter filter) 
      throws IOException {
      //get all files in the data directory
      File[] files = new File(dataDirPath).listFiles();

      for (File file : files) {
         if(!file.isDirectory()
            && !file.isHidden()
            && file.exists()
            && file.canRead()
            && filter.accept(file)
         ){
            indexFile(file);
         }
      }
      return writer.getDocStats().numDocs;
   }
}

Searcher.java

This class is used to read the indexes made on raw data and searches data using the Lucene library.

package com.tutorialspoint.lucene;

import java.io.IOException;
import java.nio.file.Paths;
import java.text.ParseException;

import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.QueryBuilder;

public class Searcher {
	
   IndexSearcher indexSearcher;
   QueryBuilder queryBuilder;
   Query query;
   
   public Searcher(String indexDirectoryPath) 
      throws IOException {
      DirectoryReader indexDirectory = DirectoryReader.open(FSDirectory.open(Paths.get(indexDirectoryPath)));
      indexSearcher = new IndexSearcher(indexDirectory);
      StandardAnalyzer analyzer = new StandardAnalyzer();
      queryBuilder = new QueryBuilder(analyzer);
   }
   
   public TopDocs search(Query query) throws IOException, ParseException {
      // Sort by a string field
      SortField fileNameSort = new SortField(LuceneConstants.FILE_NAME, SortField.Type.STRING);   
      Sort sort = new Sort(fileNameSort);
      return indexSearcher.search(query, LuceneConstants.MAX_SEARCH, sort);
   }

   public Document getDocument(ScoreDoc scoreDoc) 
      throws CorruptIndexException, IOException {
      return indexSearcher.storedFields().document(scoreDoc.doc);	
   }
}

LuceneTester.java

This class is used to test the searching capability of the Lucene library.

package com.tutorialspoint.lucene;

import java.io.IOException;
import java.text.ParseException;

import org.apache.lucene.document.Document;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;

public class LuceneTester {

   String indexDir = "D:\\lucene\\Index";
   String dataDir = "D:\\lucene\\Data";
   Indxer indexer;
   Searcher searcher;

   public static void main(String[] args) {
      LuceneTester tester;
      try {
         tester = new LuceneTester();
         tester.createIndex();
         tester.searchUsingWildCardQuery("record1*");
      } catch (IOException e) {
         e.printStackTrace();
      } catch (ParseException e) {
         e.printStackTrace();
      }
   }

   private void createIndex() throws IOException {
      indexer = new Indexer(indexDir);
      int numIndexed;
      long startTime = System.currentTimeMillis();	
      numIndexed = indexer.createIndex(dataDir, new TextFileFilter());
      long endTime = System.currentTimeMillis();
      indexer.close();
      System.out.println(numIndexed+" File indexed, time taken: "
         +(endTime-startTime)+" ms");		
   }

   private void searchUsingWildCardQuery(String searchQuery) 
      throws IOException, ParseException { 
      searcher = new Searcher(indexDir); 
      long startTime = System.currentTimeMillis(); 

      //create a term to search file name 
      Term term = new Term(LuceneConstants.FILE_NAME, searchQuery); 
      //create the term query object 
      Query query = new WildcardQuery(term); 
      //do the search 
      TopDocs hits = searcher.search(query); 
      long endTime = System.currentTimeMillis(); 

      System.out.println(hits.totalHits + 
         " documents found. Time :" + (endTime - startTime) + "ms"); 

      for(ScoreDoc scoreDoc : hits.scoreDocs) { 
         Document doc = searcher.getDocument(scoreDoc); 
         System.out.println("File: "+ doc.get(LuceneConstants.FILE_PATH)); 
      } 
   } 
}

Data & Index Directory Creation

We have used 10 text files from record1.txt to record10.txt containing names and other details of the students and put them in the directory D:\Lucene\Data. Test Data. An index directory path should be created as D:\Lucene\Index. Before running this program, delete any list of index files present in that folder.

Running the Program

Once you are done with the creation of the source, the raw data, the data directory, the index directory and the indexes, you can compile and run your program. To do this, Keep the LuceneTester.Java file tab active and use either the Run option available in the Eclipse IDE or use Ctrl + F11 to compile and run your LuceneTester application. If your application runs successfully, it will print the following message in Eclipse IDE's console −

Indexing D:\lucene\Data\record1.txt
Indexing D:\lucene\Data\record10.txt
Indexing D:\lucene\Data\record2.txt
Indexing D:\lucene\Data\record3.txt
Indexing D:\lucene\Data\record4.txt
Indexing D:\lucene\Data\record5.txt
Indexing D:\lucene\Data\record6.txt
Indexing D:\lucene\Data\record7.txt
Indexing D:\lucene\Data\record8.txt
Indexing D:\lucene\Data\record9.txt
10 File indexed, time taken: 63 ms
2 hits documents found. Time :69ms
File: D:\lucene\Data\record1.txt
File: D:\lucene\Data\record10.txt