LoginSignup
0
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

More than 3 years have passed since last update.

@rururu_kenken(nao)

ElastAlertのDockerイメージで使用しているライブラリとElastAlertを最新版にする方法

Last updated at Posted at 2020-05-28

この記事を書いた理由

bitsensor/elastalert-serverのDockerイメージがなかなか新しくならない状態。
使っているライブラリが古い。docker buildでワーニングが大量に出ているので、運用で使っていて今後も継続して利用する方向けに情報を提供する必要があると思ったため、記事を書くことにしました。

対応内容

・ElasertAlertを最新の0.2.4に変更
・nodeのバージョン変更(9.9.0 → 12.17.0)
・ライブラリ最新化
・Babel アップグレード(6 → 7)

調査中

新しいElasticsearch Node.js clientへの変更

古いElasticsearch Node.js clientが非推奨になる
https://github.com/elastic/elasticsearch-js-legacy

In the next months this client will be deprecated, 
so you should start migrating your codebase as soon as possible.
We have built a migration guide that will help you move to the new client quickly, 
and if you have questions or need help, please open an issue.

Breaking changes coming from the old client

requestとrequest-promise-native

2020年2月11日でDeprecated になっている。
axiosに変更がいいのかも

Alternative libraries to request #3143

src/controllers/rules/index.jsでrequest-promise-native使っているだけのようです。

# 5行目
import rq from 'request-promise-native';

# 150～161行目
  _downloadRules(URL) {
    const options = {
      uri: URL,
      strictSSL: false
    };
    const filename = path.basename(URL);

    return rq.get(options)
      .then(buffer => fs.outputFile(filename, buffer)
        .then(() => this._untarFile(this.rulesFolder, filename))
        .then(() => fs.remove(filename)));
  }

bitsensor/elastalert 

git clone https://github.com/bitsensor/elastalert.git

istanbul

istanbulは、Deprecated になっている。
https://www.npmjs.com/package/istanbul

scripts/test.shを削除

package.json

以下の設定を削除
"test": "./scripts/test.sh",
"istanbul": "^0.4.5",

joi

joiはメンテされてないので、@hapi/joiに変更

@hapi/joi
https://www.npmjs.com/package/@hapi/joi
https://hapi.dev/module/joi/changelog/

＜修正対象＞
・src/common/config/schema.js
・src/common/config/server_config.js
・src/handlers/test/post.js

package.json

「"joi": "^14.3.1",」を削除
「"@hapi/joi": "^17.1.1",」を追加

src/common/config/schema.js

「import Joi from 'joi';」を「import Joi from '@hapi/joi';」に変更

// Defines the config's Joi schema
import Joi from '@hapi/joi';

const schema = Joi.object().keys({
  'appName': Joi.string().default('elastalert-server'),
  'es_host': Joi.string().default('elastalert'),
  'es_port': Joi.number().default(9200),
  'writeback_index': Joi.string().default('elastalert_status'),
  'port': Joi.number().default(3030),
  'elastalertPath': Joi.string().default('/opt/elastalert'),
  'rulesPath': Joi.object().keys({
    'relative': Joi.boolean().default(true),
    'path': Joi.string().default('/rules')
  }).default(),
  'templatesPath': Joi.object().keys({
    'relative': Joi.boolean().default(true),
    'path': Joi.string().default('/rule_templates')
  }).default(),
  'dataPath': Joi.object().keys({
    'relative': Joi.boolean().default(true),
    'path': Joi.string().default('/server_data')
  }).default()
}).default();

export default schema;

src/common/config/server_config.js

「import Joi from 'joi';」を削除
「this._jsonConfig = Joi.validate(jsonConfig, schema).value;」を「this._jsonConfig = schema.validate(JSON.parse(jsonConfig)).value;」に変更

import fs from 'fs';
import path from 'path';
import schema from './schema';
import resolvePath from 'object-resolve-path';
import Logger from '../logger';

// Config file relative from project root
const configFile = 'config/config.json';
const devConfigFile = 'config/config.dev.json';

const configPath = path.join(process.cwd(), configFile);
const devConfigPath = path.join(process.cwd(), devConfigFile);
const logger = new Logger('Config');

export default class ServerConfig {
  constructor() {
    // ready() callbacks
    this._waitList = [];

    // Actual config
    this._jsonConfig = null;
  }

  /**
   * Get a value from the config.
   *
   * @param {String} key The key to load the value from.
   * @returns {*}
   */
  get(key) {
    return resolvePath(this._jsonConfig, key);
  }

  /**
   * Register a callback to call when the config is ready loading.
   *
   * @param {Function} callback The callback to register.
   */
  ready(callback) {
    this._waitList.push(callback);
  }

  /**
   * Loads the config by reading the config file or falling back to defaults.
   *
   * @returns {Promise} Returns a promise which resolves when everything is done (as a promise would).
   */
  load() {

    //TODO: Watch config file for changes and reload
    const self = this;
    return new Promise(function (resolve) {
      self._getConfig().then(function (config) {
        self._validate(config);
        resolve();
      });
    }).then(function () {
      self._waitList.forEach(function (callback) {
        callback();
      });
    });
  }

  _getConfig() {
    const self = this;

    return new Promise(function (resolve) {
      self._fileExists(devConfigPath).then(function (devConfigFound) {

        // If a dev config was found read it, otherwise check for normal config
        if (devConfigFound) {
          self._readFile(devConfigPath)
            .then(function (config) {
              resolve(config);
            })
            .catch(function () {
              resolve({});
            });
        } else {
          logger.info('Proceeding to look for normal config file.');
          self._fileExists(configPath).then(function (configFound) {
            if (configFound) {
              self._readFile(configPath)
                .then(function (config) {
                  resolve(config);
                })
                .catch(function () {
                  resolve({});
                });
            } else {
              logger.info('Using default config.');
              // If no config was found, return empty object to load defaults
              resolve({});
            }
          });
        }
      });
    });
  }

  /**
   * Checks if the config file exists and we have reading permissions
   *
   * @returns {Promise} Promise returning true if the file was found and false otherwise.
   * @private
   */
  _fileExists(filePath) {
    return new Promise(function (resolve) {
      // Check if the config file exists and has reading permissions
      try {
        fs.access(filePath, fs.F_OK | fs.R_OK, function (error) {
          if (error) {
            if (error.errno === -2) {
              logger.info(`No ${path.basename(filePath)} file was found in ${filePath}.`);
            } else {
              logger.warn(`${filePath} can't be read because of reading permission problems. Falling back to default configuration.`);
            }
            resolve(false);
          } else {
            logger.info(`A config file was found in ${filePath}. Using that config.`);
            resolve(true);
          }
        });
      } catch (error) {
        logger.error('Error getting access information with fs using `fs.access`. Error:', error);
      }
    });
  }

  /**
   * Reads the config file.
   *
   * @returns {Promise} Promise returning the config if successfully read. Rejects if reading the config failed.
   * @private
   */
  _readFile(file) {
    return new Promise(function (resolve, reject) {
      fs.readFile(file, 'utf8', function (error, config) {
        if (error) {
          logger.warn(`Unable to read config file in (${file}). Using default configuration. Error: `, error);
          reject();
        } else {
          resolve(config);
        }
      });
    });
  }

  /**
   * Validate the config using the Joi schema.
   *
   * @param {Object} jsonConfig The config to validate.
   * @private
   */
  _validate(jsonConfig) {
    // Validate the JSON config
    try {
      this._jsonConfig = schema.validate(JSON.parse(jsonConfig)).value;
    } catch (error) {
      logger.error('The config in \'config/config.json\' is not a valid config configuration. Error: ', error);
    }
  }
}

src/handlers/test/post.js

「import Joi from 'joi';」を「import Joi from '@hapi/joi';」に変更
「const validationResult = Joi.validate(request.body.options, optionsSchema);」を「const validationResult = optionsSchema.validate(request.body.options);」に変更

import RouteLogger from '../../routes/route_logger';
import {sendRequestError} from '../../common/errors/utils';
import {BodyNotSendError, RuleNotSendError, OptionsInvalidError} from '../../common/errors/test_request_errors';
import Joi from '@hapi/joi';

let logger = new RouteLogger('/test', 'POST');

const optionsSchema = Joi.object().keys({
  testType: Joi.string().valid('all', 'schemaOnly', 'countOnly').default('all'),
  days: Joi.number().min(1).default(1),
  start: Joi.string().default(''),
  end: Joi.string().default(''),
  alert: Joi.boolean().default(false),
  format: Joi.string().default(''),
  maxResults: Joi.number().default(0)
}).default();

function analyzeRequest(request) {
  if (!request.body) {
    return new BodyNotSendError();
  }

  if (!request.body.rule) {
    return new RuleNotSendError();
  }

  const validationResult = optionsSchema.validate(request.body.options);

  if (validationResult.error) {
    return new OptionsInvalidError(validationResult.error);
  }

  let body = request.body;
  body.options = validationResult.value;

  return body;
}

export default function testPostHandler(request, response) {
  /**
   * @type {ElastalertServer}
   */
  let server = request.app.get('server');
  let body = analyzeRequest(request);

  if (body.error) {
    logger.sendFailed(body.error);
    sendRequestError(response, body.error);
  }

  server.testController.testRule(body.rule, body.options)
    .then(function (consoleOutput) {
      response.send(consoleOutput);
    })
    .catch(function (consoleOutput) {
      response.status(500).send(consoleOutput);
    });
}

src/handlers/silence/post.js
このファイルはbitsensorのほうにはありません

「import Joi from 'joi';」を「import Joi from '@hapi/joi';」に変更
「const validationResult = Joi.validate(request.body, optionsSchema);」を「const validationResult = optionsSchema.validate(request.body);」に変更

import RouteLogger from '../../routes/route_logger';
import {sendRequestError} from '../../common/errors/utils';
import {BodyNotSendError, OptionsInvalidError} from '../../common/errors/silence_request_errors';
import Joi from '@hapi/joi';

let logger = new RouteLogger('/silence', 'POST');

const optionsSchema = Joi.object().keys({
  unit: Joi.string().valid('seconds', 'minutes', 'hours', 'days', 'weeks').default('minutes'),
  duration: Joi.number().min(1).default(1),
}).default();

function analyzeRequest(request) {
  if (!request.body) {
    return new BodyNotSendError();
  }

  const validationResult = optionsSchema.validate(request.body);

  if (validationResult.error) {
    return new OptionsInvalidError(validationResult.error);
  }

  let body = request.body;
  body.unit = validationResult.value.unit;
  body.duration = validationResult.value.duration;

  return body;
}

export default function silencePostHandler(request, response) {
  /**
   * @type {ElastalertServer}
   */
  let server = request.app.get('server');
  let body = analyzeRequest(request);

  if (body.error) {
    logger.sendFailed(body.error);
    sendRequestError(response, body.error);
  }

  let path = request.params.path + request.params[0];

  server.silenceController.silenceRule(path, body.unit, body.duration)
    .then(function (consoleOutput) {
      response.send(consoleOutput);
    })
    .catch(function (consoleOutput) {
      response.status(500).send(consoleOutput);
    });
}

.nvmrcを以下の内容に変更

.nvmrc
12.17.0

Makefileを以下の内容に変更

Makefile
v ?= v0.2.4

all: build

build:
	docker pull alpine:3.12 && docker pull node:14-alpine
	docker build --build-arg ELASTALERT_VERSION=$(v) -t elastalert .

server: build
	docker run -it --rm -p 3030:3030 \
	--net="host" \
	elastalert:latest

.PHONY: build

Dockerfileを以下の内容に変更

基本的にbitsensorのGitHubでプルリクエストが出ている内容です。
一部異なるのは、setup.pyをrequirements.txtと同期する対応をしているところです。

Dockerfile
FROM alpine:3.12 as py-ea
ARG ELASTALERT_VERSION=v0.2.4
ENV ELASTALERT_VERSION=${ELASTALERT_VERSION}
# URL from which to download Elastalert.
ARG ELASTALERT_URL=https://github.com/Yelp/elastalert/archive/$ELASTALERT_VERSION.zip
ENV ELASTALERT_URL=${ELASTALERT_URL}
# Elastalert home directory full path.
ENV ELASTALERT_HOME /opt/elastalert

WORKDIR /opt

RUN apk add --update --no-cache ca-certificates openssl-dev openssl python3-dev python3 py3-pip py3-yaml libffi-dev gcc musl-dev wget && \
# Download and unpack Elastalert.
    wget -O elastalert.zip "${ELASTALERT_URL}" && \
    unzip elastalert.zip && \
    rm elastalert.zip && \
    mv e* "${ELASTALERT_HOME}"

WORKDIR "${ELASTALERT_HOME}"

# Sync requirements.txt and setup.py & update py-zabbix #2818 (https://github.com/Yelp/elastalert/pull/2818)
# Install Elastalert.
RUN sed -i 's/PyYAML>=3.12/PyYAML>=5.1/g' setup.py && \
    sed -e "30i         'py-zabbix>=1.1.3'," setup.py && \
    python3 setup.py install

FROM node:14-alpine
LABEL maintainer="BitSensor <dev@bitsensor.io>"
# Set timezone for this container
ENV TZ Etc/UTC

RUN apk add --update --no-cache curl tzdata python3 make libmagic && \
    ln -s /usr/bin/python3 /usr/bin/python

COPY --from=py-ea /usr/lib/python3.8/site-packages /usr/lib/python3.8/site-packages
COPY --from=py-ea /opt/elastalert /opt/elastalert
COPY --from=py-ea /usr/bin/elastalert* /usr/bin/

WORKDIR /opt/elastalert-server
COPY . /opt/elastalert-server

RUN npm install --production --quiet
COPY config/elastalert.yaml /opt/elastalert/config.yaml
COPY config/elastalert-test.yaml /opt/elastalert/config-test.yaml
COPY config/config.json config/config.json
COPY rule_templates/ /opt/elastalert/rule_templates
COPY elastalert_modules/ /opt/elastalert/elastalert_modules

# Add default rules directory
# Set permission as unpriviledged user (1000:1000), compatible with Kubernetes
RUN mkdir -p /opt/elastalert/rules/ /opt/elastalert/server_data/tests/ \
    && chown -R node:node /opt

USER node

EXPOSE 3030
ENTRYPOINT ["npm", "start"]

.babelrcを以下の内容に変更

.babelrc
{
  "presets": [
    "@babel/preset-env"
  ]
}

index.jsを以下の内容に変更

require('babel-register'); → require('@babel/register');

index.js
require('@babel/register');
require('src');

package.jsonを以下の内容に変更

# 変更内容
 add @babel/core 7.10.2
 babel-cli               ^6.11.4  →   @babel/cli 7.10.1
 babel-preset-es2015     ^6.13.2  →   @babel/preset-env 7.10.2 
 babel-register          ^6.14.0  →   @babel/register 7.10.1
 body-parser             ^1.15.2  →   ^1.19.0 
 bunyan                   ^1.8.1  →   ^1.8.12 
 cors                     ^2.8.4  →    ^2.8.5 
 elasticsearch           ^15.1.1  →   ^16.7.1 
 express                 ^4.14.0  →   ^4.17.1 
 fs-extra                 ^5.0.0  →    ^9.0.0 
 joi                     ^13.1.2  →   ^14.3.1 
 lodash                  ^4.15.0  →  ^4.17.15 
 mkdirp                   ^0.5.1  →    ^0.5.5 
 raven                    ^2.6.1  →    ^2.6.4 
 request                 ^2.85.0  →   ^2.88.2 
 request-promise-native   ^1.0.5  →    ^1.0.8 
 tar                      ^4.4.1  →    ^6.0.2 
 ws                       ^6.0.0  →    ^7.3.0 
 eslint                  ^4.17.0  →    ^7.2.0 
 husky                   ^0.14.3  →    ^4.2.5 
 istanbul                 ^0.4.4  →    ^0.4.5 
 mocha                    ~3.0.2  →    ~8.0.1
package.json
{
  "name": "@bitsensor/elastalert",
  "version": "3.0.0-beta.0",
  "description": "A server that runs ElastAlert and exposes REST API's for manipulating rules and alerts.",
  "license": "MIT",
  "main": "index.js",
  "author": {
    "name": "BitSensor",
    "url": "https://bitsensor.io",
    "email": "dev@bitsensor.io"
  },
  "repository": {
    "type": "git",
    "url": "git+https://github.com/bitsensor/elastalert.git"
  },
  "directories": {
    "lib": "./lib",
    "test": "./test"
  },
  "dependencies": {
    "@babel/cli": "^7.10.1",
    "@babel/core": "^7.10.2",
    "@babel/preset-env": "^7.10.2",
    "@babel/register": "^7.10.1",
    "body-parser": "^1.19.0",
    "bunyan": "^1.8.12",
    "cors": "^2.8.5",
    "elasticsearch": "^16.7.1",
    "express": "^4.17.1",
    "fs-extra": "^9.0.0",
    "joi": "^14.3.1",
    "lodash": "^4.17.15",
    "mkdirp": "^1.0.4",
    "object-resolve-path": "^1.1.1",
    "randomstring": "^1.1.5",
    "raven": "^2.6.4",
    "request": "^2.88.2",
    "request-promise-native": "^1.0.8",
    "tar": "^6.0.2",
    "ws": "^7.3.0"
  },
  "devDependencies": {
    "eslint": "^7.2.0",
    "husky": "^4.2.5",
    "istanbul": "^0.4.5",
    "mocha": "~8.0.1"
  },
  "scripts": {
    "build": "babel src -d lib",
    "start": "sh ./scripts/start.sh",
    "test": "./scripts/test.sh",
    "update-authors": "./scripts/update-authors.sh",
    "precommit": "./node_modules/eslint/bin/eslint.js ."
  }
}

Dockerイメージ作成

nvm install v12.17.0

# .nvmrcで設定したversionに切り替えるにはnvm useを実行する必要があります
nvm use

docker build -t elastalert:0.2.4 .
0
0
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
0
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?