搜索引擎是如何搜索的

一個(gè)信息檢索系統(tǒng)一般有六個(gè)子系統(tǒng)組成：采選子系統(tǒng)、標(biāo)引子系統(tǒng)、詞語子系統(tǒng)、交互子系統(tǒng)、查詢子系統(tǒng)、匹配子系統(tǒng)

其中一般由采選子系統(tǒng)完成信息的收集（可以理解為爬蟲機(jī)器人），標(biāo)引子系統(tǒng)完成文檔的分詞和關(guān)鍵信息標(biāo)引（主要涉及自然語言處理技術(shù)），由詞語子系統(tǒng)建立和維護(hù)系統(tǒng)主題詞表、交互子系統(tǒng)提供用戶界面（你看到的搜索框和一些檢索配置）、查詢子系統(tǒng)負(fù)責(zé)根據(jù)搜索提問詞建立查詢式、匹配子系統(tǒng)完成查詢式和文檔集之間的信息匹配（有的可能還會(huì)有一些相關(guān)度計(jì)算以根據(jù)查詢進(jìn)行排序）及結(jié)果返回。

以上就是一個(gè)信息檢索系統(tǒng)的一般模型，具體的實(shí)現(xiàn)每個(gè)搜索引擎和可能會(huì)有不同。

我們知道網(wǎng)頁其實(shí)就是一個(gè)個(gè)超文本文檔。根據(jù)以上一般原理，我們可以想象，類似的搜索引擎就是完成了網(wǎng)頁的抓取、分析、處理、存儲(chǔ)、檢索、排序、返回，接著你就可以看到你和你搜索的關(guān)鍵詞匹配的結(jié)果了。

你有興趣的話，可以找一些關(guān)于信息檢索與處理相關(guān)的書看一看，GitHub上也有一些開源的搜索引擎項(xiàng)目可供參考。

網(wǎng)站的話現(xiàn)在不是要流行分布式搜索嗎那個(gè)，elasticSearch可以學(xué)學(xué)

這個(gè)有點(diǎn)說來話長，你先看下百度百科把基本原理搞懂，下面代碼是基于lucene的搜索：

package com.tray.indexData;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.math.BigInteger;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
  
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Fieldable;
import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Term;
import org.apache.lucene.queryParser.MultiFieldQueryParser;
import org.apache.lucene.queryParser.QueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PrefixQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.Version;
import org.wltea.analyzer.lucene.IKAnalyzer;
  
import com.tray.bean.SerachResult;
import com.tray.common.tools.DateFormater;
  
public class LuceneSearch {
      
    private static String DISC_URL = "/home/indexData/data";
      
    static {
        String os = System.getProperty("os.name"); 
        if(os.toLowerCase().startsWith("win")){ 
            DISC_URL = "E:\\indexData\\data";
        }
        else{
            DISC_URL ="/home/indexData/data";
        }
    }
          
    //指定分詞器
    private Analyzer analyzer=new IKAnalyzer();
    private static Directory directory;
    //配置
    private static IndexWriterConfig iwConfig;
    //配置IndexWriter
    private static IndexWriter writer; 
    private static File indexFile = null; 
      
    private static Version version = Version.LUCENE_36;
      
    private final int PAPGESIZE=10;
  
    /**
     * 全量索引
     * @Author haoning
     */
    public void init() throws Exception {
          
        try {
            indexFile = new File(DISC_URL);
            if (!indexFile.exists()) {
                indexFile.mkdir();
            }
            directory=FSDirectory.open(indexFile); 
            //配置IndexWriterConfig 
            iwConfig = new IndexWriterConfig(version,analyzer); 
            iwConfig.setOpenMode(OpenMode.CREATE_OR_APPEND); 
                //創(chuàng)建寫索引對象 
            writer = new IndexWriter(directory,iwConfig);  
        } catch (Exception e) {
        }
    }
      
    public void closeWriter(){
        try {
            writer.close();
        } catch (CorruptIndexException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
      
    public void commit(){
          
        try {
            writer.commit();
        } catch (CorruptIndexException e) {
            e.printStackTrace();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
      
    /**
     * 一個(gè)一個(gè)索引
     * @Author haoning
     */
    public void singleIndex(Document doc) throws Exception {
        writer.addDocument(doc);
    }
      
    /**
     * 一個(gè)跟新
     * @Author haoning
     */
    public void singleUpdate(Document doc) throws Exception {
        Term term = new Term("url", doc.get("url"));
        writer.updateDocument(term,doc);
    }
      
    /**
     * 全量索引
     * @Author haoning
     */
    public void fullIndex(Document[] documentes) throws Exception {
          
        writer.deleteAll();
        for (Document document : documentes) {
            writer.addDocument(document);
        }
        writer.commit();
    }
      
    /**
     * 根據(jù)id刪除索引
     * @Author haoning
     */
    public void deleteIndex(Document document)throws Exception{
        Term term = new Term("url", document.get("url"));//url才是唯一標(biāo)志
        writer.deleteDocuments(term);
        writer.commit();
    }
      
    /**
     * 根據(jù)id增量索引
     * @Author haoning
     */
    public void updateIndex(Document[] documentes) throws Exception{
        for (Document document : documentes) {
            Term term = new Term("url", document.get("url"));
            writer.updateDocument(term, document);
        }
        writer.commit();
    }
      
    /**
     * 直接查詢
     * @Author haoning
     */
    public void simpleSearch(String filedStr,String queryStr,int page, int pageSize) throws Exception{
        File indexDir = new File(DISC_URL); 
        //索引目錄 
        Directory dir=FSDirectory.open(indexDir); 
        //根據(jù)索引目錄創(chuàng)建讀索引對象 
        IndexReader reader = IndexReader.open(dir); 
        //搜索對象創(chuàng)建 
        IndexSearcher searcher = new IndexSearcher(reader);
        TopScoreDocCollector topCollector = TopScoreDocCollector.create(searcher.maxDoc(), false);
          
        Term term = new Term(filedStr, queryStr);
        Query query = new TermQuery(term);
        searcher.search(query, topCollector);
        ScoreDoc[] docs = topCollector.topDocs((page-1)*pageSize, pageSize).scoreDocs;
          
        printScoreDoc(docs, searcher);
    }
      
    /**
     * 高亮查詢
     * @Author haoning
     */
    public Map<String, Object> highLightSearch(String filed,String keyWord,int curpage, int pageSize) throws Exception{
        List<SerachResult> list=new ArrayList<SerachResult>();
        Map<String,Object> map = new HashMap<String,Object>();
        if (curpage <= 0) {
            curpage = 1;
        }
        if (pageSize <= 0 || pageSize>20) {
             pageSize = PAPGESIZE;
        }
        File indexDir = new File(DISC_URL); //索引目錄  
        Directory dir=FSDirectory.open(indexDir);//根據(jù)索引目錄創(chuàng)建讀索引對象   
        IndexReader reader = IndexReader.open(dir);//搜索對象創(chuàng)建   
        IndexSearcher searcher = new IndexSearcher(reader);
          
        int start = (curpage - 1) * pageSize;
          
        Analyzer analyzer = new IKAnalyzer(true);
        QueryParser queryParser = new QueryParser(Version.LUCENE_36, filed, analyzer);
        queryParser.setDefaultOperator(QueryParser.AND_OPERATOR);
        Query query = queryParser.parse(keyWord);
          
        int hm = start + pageSize;
        TopScoreDocCollector res = TopScoreDocCollector.create(hm, false);
        searcher.search(query, res);
          
        SimpleHTMLFormatter simpleHTMLFormatter = new SimpleHTMLFormatter("<span style='color:red'>", "</span>");
        Highlighter highlighter = new Highlighter(simpleHTMLFormatter, new QueryScorer(query));
          
        long amount = res.getTotalHits();
        //long pages = (rowCount - 1) / pageSize + 1; //計(jì)算總頁數(shù)
          
        map.put("amount",amount);//總共多少條記錄
          
        TopDocs tds = res.topDocs(start, pageSize);
        ScoreDoc[] sd = tds.scoreDocs;
          
        for (int i = 0; i < sd.length; i++) {
            Document doc = searcher.doc(sd[i].doc);
            String temp=doc.get("name");
            //做高亮處理
            TokenStream ts = analyzer.tokenStream("name", new StringReader(temp));
              
            SerachResult record=new SerachResult();
            String name = highlighter.getBestFragment(ts,temp);
            String skydirverName=doc.get("skydirverName");
            String username=doc.get("username");
            String shareTime=doc.get("shareTime");
            String describ=doc.get("describ");
            String typeId=doc.get("typeId");
            String id=doc.get("id");
            String url=doc.get("url");
              
            record.setName(name);
            record.setSkydriverName(skydirverName);
            record.setUsername(username);
            record.setShareTime(DateFormater.getFormatDate(shareTime,"yyyy-MM-dd HH:mm:ss"));
            record.setDescrib(describ);
            record.setTypeId(Integer.parseInt(typeId));
            record.setId(new BigInteger(id));
            record.setUrl(url);
            list.add(record);
              
            /*System.out.println("name:"+name);
            System.out.println("skydirverName:"+skydirverName);
            System.out.println("username:"+username);
            System.out.println("shareTime:"+shareTime);
            System.out.println("describ:"+describ);
            System.out.println("typeId:"+typeId);
            System.out.println("id:"+id);
            System.out.println("url:"+url);*/
        }
        map.put("source",list);
        return map;
    }
      
    /**
     * 根據(jù)前綴查詢
     * @Author haoning
     */
    public void prefixSearch(String filedStr,String queryStr) throws Exception{
        File indexDir = new File(DISC_URL); 
        //索引目錄 
        Directory dir=FSDirectory.open(indexDir); 
        //根據(jù)索引目錄創(chuàng)建讀索引對象 
        IndexReader reader = IndexReader.open(dir); 
        //搜索對象創(chuàng)建 
        IndexSearcher searcher = new IndexSearcher(reader);
          
        Term term = new Term(filedStr, queryStr);
        Query query = new PrefixQuery(term);
          
        ScoreDoc[] docs = searcher.search(query, 3).scoreDocs;
        printScoreDoc(docs, searcher);
    }
      
    /**
     * 通配符查詢
     * @Author haoning
     */
    public void wildcardSearch(String filedStr,String queryStr) throws Exception{
        File indexDir = new File(DISC_URL); 
        //索引目錄 
        Directory dir=FSDirectory.open(indexDir); 
        //根據(jù)索引目錄創(chuàng)建讀索引對象 
        IndexReader reader = IndexReader.open(dir); 
        //搜索對象創(chuàng)建 
        IndexSearcher searcher = new IndexSearcher(reader);
          
        Term term = new Term(filedStr, queryStr);
        Query query = new WildcardQuery(term);
        ScoreDoc[] docs = searcher.search(query, 3).scoreDocs;
        printScoreDoc(docs, searcher);
    }
      
    /**
     * 分詞查詢
     * @Author haoning
     */
    public void analyzerSearch(String filedStr,String queryStr) throws Exception{
        File indexDir = new File(DISC_URL); 
        //索引目錄 
        Directory dir=FSDirectory.open(indexDir); 
        //根據(jù)索引目錄創(chuàng)建讀索引對象 
        IndexReader reader = IndexReader.open(dir); 
        //搜索對象創(chuàng)建 
        IndexSearcher searcher = new IndexSearcher(reader);
          
        QueryParser queryParser = new QueryParser(version, filedStr, analyzer);
        Query query = queryParser.parse(queryStr);
          
        ScoreDoc[] docs = searcher.search(query, 3).scoreDocs;
        printScoreDoc(docs, searcher);
    }
      
    /**
     * 多屬性分詞查詢
     * @Author haoning
     */
    public void multiAnalyzerSearch(String[] filedStr,String queryStr) throws Exception{
        File indexDir = new File(DISC_URL); 
        //索引目錄 
        Directory dir=FSDirectory.open(indexDir); 
        //根據(jù)索引目錄創(chuàng)建讀索引對象 
        IndexReader reader = IndexReader.open(dir); 
        //搜索對象創(chuàng)建 
        IndexSearcher searcher = new IndexSearcher(reader);
        QueryParser queryParser = new MultiFieldQueryParser(version, filedStr, analyzer);
        Query query = queryParser.parse(queryStr);
          
        ScoreDoc[] docs = searcher.search(query, 3).scoreDocs;
        printScoreDoc(docs, searcher);
    }
      
    public void printScoreDoc(ScoreDoc[] docs,IndexSearcher searcher)throws Exception{
        for (int i = 0; i < docs.length; i++) {
            List<Fieldable> list = searcher.doc(docs[i].doc).getFields();
            for (Fieldable fieldable : list) {
                String fieldName = fieldable.name();
                String fieldValue = fieldable.stringValue();
                System.out.println(fieldName+" : "+fieldValue);
            }
        }
    }
}
　　

你可以看下這篇知乎專欄：
https://zhuanlan.zhihu.com/p/...