elasticsearchとdatatablesの連携

要件

1. elasticsearchでの検索結果をjqueryのdatatablesで出力したい

環境

• Go 1.3 (with revel as Web Application Framework)
• elasticsearch 1.3.x (with Kuromoji)
• jquery 1.10.x
• datatables 1.10.x

背景

もともとmongoをうらのDBとして利用していましたが、日本語での全文検索の精度がどうしても悪いのと、数十万件規模のレコードの検索に3秒ほど時間がかかってしまったのでelasticsearchを試してみることにしました。

お断り

snippetは必要な部分のみ抽出してます。
structで宣言しておきながらつかってないのもあるかも。。。

snipet

lib.go

type ReturnDataTable struct {
	Draw               int                      `json:"draw"`
	RecordsTotal       int                      `json:"recordsTotal"`
	RecordsFiltered    interface{}              `json:"recordsFiltered"`
	Data               []map[string]interface{} `json:"data"`
	SearchString       string
	Length             int
	OrderColumn        string
	OrderDir           int
	SearchFields       []string
	Start              int
	AndColumns         []bson.M
	ObjectIdFields     []string
	DefaultOrderColumn string
}

type ESClause struct {
	From      int
	Size      int
	Query     string
	SortField string
	SortOrder string
	Draw      int
}

type ESFilter struct {
	Term map[string]interface{} `json:"term"`
}

type ESFilterNot struct {
	Term map[string]interface{} `json:"term"`
}

type ESFilterShould struct {
	Term map[string]interface{} `json:"term"`
}

type ESFilterMustNot struct {
	Term map[string]interface{} `json:"term"`
}

func ESGet(clause *ESClause, should *[]ESFilterShould, must_not *[]ESFilterMustNot) (*ReturnDataTable, error) {
	var err error
	var ret *ReturnDataTable

	url_base, _ := revel.Config.String("es_url")

	type FilterBool struct {
		Should  *[]ESFilterShould  `json:"should"`
		MustNot *[]ESFilterMustNot `json:"must_not"`
	}

	type FilterNotBox struct {
		Terms *[]ESFilterNot `json:"and"`
	}

	type FilterBox struct {
		FilterBool `json:"bool"`
	}

	type QueryString struct {
		Query string `json:"query"`
		//Analyzer string `json:"analyzer"`
	}

	type Query struct {
		QueryString `json:"query_string"`
	}

	type Size struct {
		Size int `json:"size"`
	}

	type From struct {
		From int `json:"from"`
	}

	type Analyzer struct {
		Analyzer string `json:"analyzer"`
	}

	type JsonBucket struct {
		//QueryString `json:",inline"`
		From      `json:",inline"`
		Size      `json:",inline"`
		Query     `json:"query"`
		FilterBox `json:"filter"`
	}

	//qstr := QueryString{clause.Query, ""}
	qstr := QueryString{clause.Query}
	qs := Query{qstr}

	filbool := FilterBool{should, must_not}
	filbox := FilterBox{filbool}

	bucket := JsonBucket{
		From{clause.From},
		Size{clause.Size},
		qs,
		filbox,
	}

	//revel.ERROR.Println(filbox)

	jbucket, _ := json.Marshal(bucket)

	if err != nil {
		revel.ERROR.Println(err)
	}

	url := url_base

	client := &http.Client{}
	req, _ := http.NewRequest(
		"POST",
		url,
		bytes.NewReader(jbucket),
	)

	req.Header.Set("Content-Type", "application/json; charset=UTF-8")

	resp, _ := client.Do(req)
	body, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	var f interface{}
	json.Unmarshal(body, &f)
	m := f.(map[string]interface{})

	n, ok := m["hits"].(map[string]interface{})

	if !ok {

		ret = &ReturnDataTable{}

		ret.Draw = clause.Draw
		ret.Data = make([]map[string]interface{}, 0)
		ret.RecordsFiltered = 0
		ret.Start = clause.From

		return ret, err

	}

	var fs []interface{}

	fs = n["hits"].([]interface{})

	var sources []map[string]interface{}

	for _, v := range fs {
		d := v.(map[string]interface{})
		sources = append(sources, d["_source"].(map[string]interface{}))
	}

	ifound := len(sources)

	var retVals []map[string]interface{}

	for _, v := range sources {

		var retVal map[string]interface{}

		var id string

		retVal = make(map[string]interface{})

		for k, nv := range v {
			retVal[k] = nv
			if k == "_id" {
				id = nv.(string)
			}
		}
		retVal["Id"] = id
		retVal["DT_RowId"] = id
		retVal["DT_RowClass"] = "Rest_" + id
		retVal["DT_RowData"] = map[string]string{"id": id}
		retVals = append(retVals, retVal)
	}

	if ifound == 0 {
		retVals = make([]map[string]interface{}, 0)
	}

	ret = &ReturnDataTable{}

	ret.Draw = clause.Draw
	ret.Data = retVals
	ret.RecordsFiltered = n["total"]
	ret.Start = clause.From

	return ret, err

}

controllers/app.go

func (c Restaurant) GetDataTable() revel.Result {
	var clause *lib.ESClause
	var should []lib.ESFilterShould = []library.ESFilterShould{}
	var must_not []lib.ESFilterMustNot = []library.ESFilterMustNot{}

    // set query clauses here
	iLen, _ := strconv.Atoi(c.Params.Get("length"))
	sVal := c.Params.Get("search[value]")
    /* your custom conditions
	bSomeFlag := c.Params.Get("someflag")
	_should := lib.ESFilterShould{map[string]interface{}{"whateverflag": true}}
	//_must_not := lib.ESFilterMustNot{map[string]interface{}{"someeverflag": false}}

    */

	if sVal == "" {
		sVal = "*"
	}

    // append conditions to "should" and/or "must_not" if needed

	var iDraw int = 0
	c.Params.Bind(&iDraw, "draw")

	var iFrom int = 0
	c.Params.Bind(&iFrom, "start")

	clause = &lib.ESClause{
		Query: sVal,
		Size:  iLen,
		From:  iFrom,
		Draw:  iDraw,
	}

	ret, err := lib.ESGet(clause, &should, &must_not)

	if err != nil {
		revel.INFO.Println(err)
	}

	return c.RenderJson(ret)
}

app.js

  var dt = $("#table").dataTable({
    "dom": '<"row-fluid"<"span2"l><"span10"f<"bushi-dt-btn2">>>tip',
    "bLengthChange": true,
    "iDisplayLength": 25,
    "sPaginationType": "bootstrap", 
    "showPagenation": true,
    "showFilter": true,
    "info": true,
    "bDestroy": true,
    "processing": true,
    "serverSide": true,
    "ajax": {
      url: "/app/get_dt", // the uri for controller/app
      data: { "whateverflag": "true" }, // default param you want to send
      complete: function(d) {
        $("#loading").hide();
      },
    });

効果

実測値で検索実行時間は0.6秒ほど。精度はelasticsearchのIndex構築手法によりけりになってしまいますが、利用してくれてるユーザからの満足度はそこそこ高いです。

ご質問などありましたらコメントにどうぞ！