Strapi Advent Calenderの1日目はStrapi公式ブログのCSVからデータをインポートするプラグインを作成する記事のpart3内の、CSVファイルからデータをインポートする機能までを日本語でまとめてみました。
この記事はStrapiでCSVからデータをインポートするプラグインを作るの続きになります。また、公式ブログのパート2は割愛しています。
1. ファイルの解析リクエストの応答処理を作成する
import-content/services
にutils
ディレクトリを追加し、その中にutils.js
を作成して以下を記述します。
"use strict";
const request = require("request");
const contentTypeParser = require("content-type-parser");
const RssParser = require("rss-parser");
const CsvParser = require("csv-parse/lib/sync");
const urlRegEx = /((([A-Za-z]{3,9}:(?:\/\/)?)(?:[\-;:&=\+\$,\w]+@)?[A-Za-z0-9\.\-]+|(?:www\.|[\- ;:&=\+\$,\w]+@)[A-Za-z0-9\.\-]+)((?:\/[\+~%\/\.\w\-]*)?\??(?:[\-\+=&;%@\.\w]*)#?(?:[\.\!\/\\\w]*))?)/g;
const URL_REGEXP = new RegExp(urlRegEx);
const validateUrl = url => {
URL_REGEXP.lastIndex = 0;
return URL_REGEXP.test(url);
};
const EMAIL_REGEXP = /^(([^<>()\[\]\\.,;:\s@"]+(\.[^<>()\[\]\\.,;:\s@"]+)*)|(".+"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$/;
const stringIsEmail = data => {
EMAIL_REGEXP.lastIndex = 0;
return EMAIL_REGEXP.test(data);
};
const getDataFromUrl = url => {
return new Promise((resolve, reject) => {
if (!validateUrl(url)) return reject("invalid URL");
request(url, null, async (err, res, body) => {
if (err) {
reject(err);
}
resolve({ dataType: res.headers["content-type"], body });
});
});
};
const resolveDataFromRequest = async ctx => {
const { source, type, options, data } = ctx.request.body;
switch (source) {
case "upload":
return { dataType: type, body: data, options };
case "url":
const { dataType, body } = await getDataFromUrl(options.url);
return { dataType, body, options };
case "raw":
return {
dataType: type,
body: options.rawText,
options
};
}
};
const getItemsFromData = ({ dataType, body, options }) =>
new Promise(async (resolve, reject) => {
const parsedContentType = contentTypeParser(dataType);
if (parsedContentType.isXML()) {
const parser = new RssParser();
const feed = await parser.parseString(body);
return resolve({ sourceType: "rss", items: feed.items });
}
if (dataType === "text/csv" || dataType === "application/vnd.ms-excel") {
const items = CsvParser(body, {
...options,
columns: true
});
return resolve({ sourceType: "csv", items });
}
reject({
contentType: parsedContentType.toString()
});
});
const urlIsMedia = url => {
try {
const parsed = new URL(url);
const extension = parsed.pathname
.split(".")
.pop()
.toLowerCase();
switch (extension) {
case "png":
case "gif":
case "jpg":
case "jpeg":
case "svg":
case "bmp":
case "tif":
case "tiff":
return true;
case "mp3":
case "wav":
case "ogg":
return true;
case "mp4":
case "avi":
return true;
default:
return false;
}
} catch (error) {
return false;
}
};
module.exports = {
resolveDataFromRequest,
getItemsFromData,
getDataFromUrl,
stringIsEmail,
urlIsMedia
};
次にimport-content/services/utils
のなかにfieldUtils.js
を作成し、以下を記述します。
const getUrls = require("get-urls");
const { urlIsMedia, stringIsEmail } = require("./utils");
const striptags = require("striptags");
const detectStringFieldFormat = data => {
if (new Date(data).toString() !== "Invalid Date") return "date";
if (stringIsEmail(data)) return "email";
if (data.length !== striptags(data).length) {
return "xml";
}
return "string";
};
const detectFieldFormat = data => {
switch (typeof data) {
case "number":
return "number";
case "boolean":
return "boolean";
case "object":
return "object";
case "string":
return detectStringFieldFormat(data);
}
};
const compileStatsForFieldData = fieldData => {
const stats = {};
switch (typeof fieldData) {
case "string":
try {
const urls = Array.from(getUrls(fieldData));
const l = urls.length;
for (let i = 0; i < l; ++i) {
if (urlIsMedia(urls[i])) {
stats.hasMediaUrls = true;
break;
}
}
} catch (e) {
console.log(e);
}
stats.length = fieldData.length;
break;
case "object":
if (urlIsMedia(fieldData.url)) {
stats.hasMediaUrls = true;
}
stats.length = JSON.stringify(fieldData).length;
break;
default:
console.log(typeof fieldData, fieldData);
}
stats.format = detectFieldFormat(fieldData);
return stats;
};
const getMediaUrlsFromFieldData = fieldData => {
switch (typeof fieldData) {
case "string":
return Array.from(getUrls(fieldData)).filter(urlIsMedia);
case "object":
return urlIsMedia(fieldData.url) ? [fieldData.url] : [];
}
};
module.exports = {
detectStringFieldFormat,
detectFieldFormat,
compileStatsForFieldData,
getMediaUrlsFromFieldData
};
今度は、import-content/services/utils
のなかにanalyzer.js
を作成し、以下を記述します。
"use strict";
const _ = require("lodash");
var ss = require("simple-statistics");
const { compileStatsForFieldData } = require("./fieldUtils");
const getFieldNameSet = items => {
const fieldNames = new Set();
items.forEach(item => {
try {
Object.keys(item).forEach(fieldName => fieldNames.add(fieldName));
} catch (e) {
console.log(e);
}
});
return fieldNames;
};
const analyze = (sourceType, items) => {
const fieldNames = getFieldNameSet(items);
const fieldAnalyses = {};
fieldNames.forEach(fieldName => (fieldAnalyses[fieldName] = []));
items.forEach(item => {
fieldNames.forEach(fieldName => {
const fieldData = item[fieldName];
const fieldStats = compileStatsForFieldData(fieldData);
fieldAnalyses[fieldName].push(fieldStats);
});
});
const fieldStats = Object.keys(fieldAnalyses).map(fieldName => {
const fieldAnalysis = fieldAnalyses[fieldName];
const fieldStat = { fieldName, count: fieldAnalysis.length };
try {
fieldStat.format = _.chain(fieldAnalysis)
.countBy("format")
.map((value, key) => ({ count: value, type: key }))
.sortBy("count")
.reverse()
.head()
.get("type")
.value();
} catch (e) {
console.log(e);
}
fieldStat.hasMediaUrls = fieldAnalysis.some(fa => Boolean(fa.hasMediaUrls));
const lengths = _.map(fieldAnalysis, "length");
fieldStat.minLength = ss.min(lengths);
fieldStat.maxLength = ss.max(lengths);
fieldStat.meanLength = ss.mean(lengths).toFixed(2);
return fieldStat;
});
return { itemCount: items.length, fieldStats };
};
module.exports = { getFieldNameSet, analyze };
plugins/import-content/services/import-content.js
を開いて、先ほど作成したサービスを読み込みます。
"use strict";
/** * ImportContent.js service
* * @description: A set of functions similar to controller's actions to avoid code duplication. */
const { resolveDataFromRequest, getItemsFromData } = require("./utils/utils");
const analyzer = require("./utils/analyzer");
module.exports = {
preAnalyzeImportFile: async ctx => {
const { dataType, body, options } = await resolveDataFromRequest(ctx);
const { sourceType, items } = await getItemsFromData({
dataType,
body,
options
});
const analysis = analyzer.analyze(sourceType, items);
return { sourceType, ...analysis };
}
};
サービスを使用する準備ができたので、import-content/controllers/import-content.js
で使用するように記述していきます。
"use strict";
module.exports = {
preAnalyzeImportFile: async ctx => {
const services = strapi.plugins["import-content"].services;
try {
const data = await services["importcontent"].preAnalyzeImportFile(ctx);
ctx.send(data);
} catch (error) {
console.log(error);
ctx.response.status = 406;
ctx.response.message = "could not parse: " + error;
}
}
};
次に、plugins/import-content/config/routes.json
を以下のように書き換えます。
{
"routes": [
{
"method": "POST",
"path": "/preAnalyzeImportFile",
"handler": "import-content.preAnalyzeImportFile",
"config": {
"policies": []
}
}
]
}
ここまで作成して、UI上でCSVファイルを選択しAnalyzeボタンを押すと、Analyzed Successfully
と表示されるようになります。
次は、解析したファイルの項目と、インポート先のモデルの項目とを紐づける機能を作成していきます。
2. 解析したファイルの項目と、インポート先のモデルの項目とを紐づける機能を作成する
import-content/admin/src/components
にMappingTable
ディレクトリを追加し、そのなかにTargetFieldSelect.js
を作成し、以下を記述します。
import React, { Component } from "react";
import { Select } from "@buffetjs/core";
import { get } from "lodash";
class TargetFieldSelect extends Component {
state = {
selectedTarget: ""
};
componentDidMount() {
const options = this.fillOptions();
this.setState({ selectedTarget: options && options[0] });
}
onChange(selectedTarget) {
this.props.onChange(selectedTarget);
this.setState({ selectedTarget });
}
fillOptions() {
const { targetModel } = this.props;
const schemaAttributes = get(targetModel, ["schema", "attributes"], {});
const options = Object.keys(schemaAttributes)
.map(fieldName => {
const attribute = get(schemaAttributes, [fieldName], {});
return attribute.type && { label: fieldName, value: fieldName };
})
.filter(obj => obj !== undefined);
return [{ label: "None", value: "none" }, ...options];
}
render() {
return (
<Select
name={"targetField"}
value={this.state.selectedTarget}
options={this.fillOptions()}
onChange={({ target: { value } }) => this.onChange(value)}
/>
);
}
}
export default TargetFieldSelect;
次に、MappingTable
ディレクトリ内にMappingOptions.js
を作成し、以下を記述します。
import React, { Component } from "react";
import TargetFieldSelect from "./TargetFieldSelect";
import { Label } from "@buffetjs/core";
const MappingOptions = ({ stat, onChange, targetModel }) => {
return (
<div>
{stat.format === "xml" && (
<div>
<Label htmlFor={"stripCheckbox"} message={"Strip Tags"} />
<input
name={"stripCheckbox"}
type="checkbox"
onChange={e => onChange({ stripTags: e.target.checked })}
/>
</div>
)}
{stat.hasMediaUrls && (
<div style={{ paddingTop: 8, paddingBottom: 8 }}>
<Label
htmlFor={"mediaTargetSelect"}
message={"Import Media to Field"}
/>
<TargetFieldSelect
name={"mediaTargetSelect"}
targetModel={targetModel}
onChange={targetField =>
onChange({ importMediaToField: targetField })
}
/>
</div>
)}
</div>
);
};
export default MappingOptions;
同じくMappingTable
ディレクトリ内にindex.js
を作成し、以下を記述します。
import React, { Component } from "react";
import PropTypes from "prop-types";
import MappingOptions from "./MappingOptions";
import TargetFieldSelect from "./TargetFieldSelect";
import _ from "lodash";
import Row from "../Row";
import { Table } from "@buffetjs/core";
import {
Bool as BoolIcon,
Json as JsonIcon,
Text as TextIcon,
NumberIcon,
Email as EmailIcon,
Calendar as DateIcon,
RichText as XmlIcon
} from "@buffetjs/icons";
class MappingTable extends Component {
state = { mapping: {} };
CustomRow = ({ row }) => {
const { fieldName, count, format, minLength, maxLength, meanLength } = row;
return (
<tr style={{ paddingTop: 18 }}>
<td>{fieldName}</td>
<td>
<p>{count}</p>
</td>
<td>
{format === "string" && <TextIcon fill="#fdd835" />}
{format === "number" && <NumberIcon fill="#fdd835" />}
{format === "boolean" && <BoolIcon fill="#fdd835" />}
{format === "object" && <JsonIcon fill="#fdd835" />}
{format === "email" && <EmailIcon fill="#fdd835" />}
{format === "date" && <DateIcon fill="#fdd835" />}
{format === "xml" && <XmlIcon fill="#fdd835" />} <p>{format}</p>
</td>
<td>
<span>{minLength}</span>
</td>
<td>
<p>{maxLength}</p>
</td>
<td>
<p>{meanLength}</p>
</td>
<td>
<MappingOptions
targetModel={this.props.targetModel}
stat={row}
onChange={this.changeMappingOptions(row)}
/>
</td>
<td>
{this.props.targetModel && (
<TargetFieldSelect
targetModel={this.props.targetModel}
onChange={targetField => this.setMapping(fieldName, targetField)}
/>
)}
</td>
</tr>
);
};
changeMappingOptions = stat => options => {
let newState = _.cloneDeep(this.state);
for (let key in options) {
_.set(newState, `mapping[${stat.fieldName}][${key}]`, options[key]);
}
this.setState(newState, () => this.props.onChange(this.state.mapping));
};
setMapping = (source, targetField) => {
const state = _.set(
this.state,
`mapping[${source}]['targetField']`,
targetField
);
this.setState(state, () => this.props.onChange(this.state.mapping));
console.log(this.state.mapping);
};
render() {
const { analysis } = this.props;
const props = {
title: "Field Mapping",
subtitle:
"Configure the Relationship between CSV Fields and Content type Fields"
};
const headers = [
{ name: "Field", value: "fieldName" },
{ name: "Count", value: "count" },
{ name: "Format", value: "format" },
{ name: "Min Length", value: "minLength" },
{ name: "Max Length", value: "maxLength" },
{ name: "Mean Length", value: "meanLength" },
{ name: "Options", value: "options" },
{ name: "Destination", value: "destination" }
];
const items = [...analysis.fieldStats];
return (
<Table
{...props}
headers={headers}
rows={items}
customRow={this.CustomRow}
/>
);
}
}
MappingTable.propTypes = {
analysis: PropTypes.object.isRequired,
targetModel: PropTypes.object,
onChange: PropTypes.func
};
export default MappingTable;
MappingTable
をHomePage
で使用するために、インポート文の追加、stateの項目の追加、メソッドの追加、render()に追記をします。
import React, { memo, Component } from "react";
import {request} from "strapi-helper-plugin";
import PropTypes from "prop-types";
import pluginId from "../../pluginId";
import UploadFileForm from "../../components/UploadFileForm";
import {
HeaderNav,
LoadingIndicator,
PluginHeader
} from "strapi-helper-plugin";
import Row from "../../components/Row";
import Block from "../../components/Block";
import { Select, Label } from "@buffetjs/core";
import { get, has, isEmpty, pickBy, set } from "lodash";
import MappingTable from "../../components/MappingTable"; // 追加
import { Button } from "@buffetjs/core"; // 追加
const getUrl = to =>
to ? `/plugins/${pluginId}/${to}` : `/plugins/${pluginId}`;
class HomePage extends Component {
importSources = [
{ label: "External URL ", value: "url" },
{ label: "Upload file", value: "upload" },
{ label: "Raw text", value: "raw" }
];
state = {
loading: true,
modelOptions: [],
models: [],
importSource: "upload",
analyzing: false,
analysis: null,
selectedContentType: "",
fieldMapping: {} // 追加
};
onSaveImport = async () => { // 追加
const { selectedContentType, fieldMapping } = this.state;
const { analysisConfig } = this;
const importConfig = {
...analysisConfig,
contentType: selectedContentType,
fieldMapping
};
try {
await request("/import-content", { method: "POST", body: importConfig });
this.setState({ saving: false }, () => {
strapi.notification.info("Import started");
});
} catch (e) {
strapi.notification.error(`${e}`);
}
};
getTargetModel = () => { // 追加
const { models } = this.state;
if (!models) return null;
return models.find(model => model.uid === this.state.selectedContentType);
};
setFieldMapping = fieldMapping => { // 追加
this.setState({ fieldMapping });
};
~~省略~~
render() {
return (
<div className={"container-fluid"} style={{ padding: "18px 30px" }}>
<PluginHeader
title={"Import Content"}
description={"Import CSV and RSS-Feed into your Content Types"}
/>
<HeaderNav
links={[
{
name: "Import Data",
to: getUrl("")
},
{
name: "Import History",
to: getUrl("history")
}
]}
style={{ marginTop: "4.4rem" }}
/>
<div className="row">
~~省略~~
</div>
{this.state.analysis && ( // 追加
<Row className="row">
<MappingTable
analysis={this.state.analysis}
targetModel={this.getTargetModel()}
onChange={this.setFieldMapping}
/>
<Button
style={{ marginTop: 12 }}
label={"Run the Import"}
onClick={this.onSaveImport}
/>
</Row>
)}
</div>
);
}
}
export default memo(HomePage);
}
ここまでで、GUI上でCSVファイルを選択してAnalyzeボタンを押すと、CSVファイルの項目とインポート先のモデルの項目を紐づけるエリアが表示されるようになります。
次は、Run the Import
ボタンを押した後の、インポート処理を実装していきます。
インポート処理を実装する
import-content/services/utils
のなかにimportFields.jsを作成し、以下を記述します。
const striptags = require("striptags");
const importFields = async (sourceItem, fieldMapping) => {
const importedItem = {};
Object.keys(fieldMapping).forEach(async sourceField => {
const { targetField, stripTags } = fieldMapping[sourceField];
if (!targetField || targetField === "none") {
return;
}
const originalValue = sourceItem[sourceField];
importedItem[targetField] = stripTags
? striptags(originalValue)
: originalValue;
});
return importedItem;
};
module.exports = importFields;
次に、import-content/services/utils
のなかにfileFromBuffer.jsを作成し、以下を記述します。
const crypto = require('crypto')
const uuid = require("uuid/v4");
function niceHash(buffer) {
return crypto
.createHash("sha256")
.update(buffer)
.digest("base64")
.replace(/=/g, "")
.replace(/\//g, "-")
.replace(/\+/, "_");
}
const fileFromBuffer = (mimeType, extension, buffer) => {
const fid = uuid();
return {
buffer,
sha256: niceHash(buffer),
hash: fid.replace(/-/g, ""),
name: `${fid}.${extension}`,
ext: `.${extension}`,
mime: mimeType,
size: (buffer.length / 1000).toFixed(2)
};
};
module.exports = fileFromBuffer;
import-content/services/utils
のなかにimportMediaFiles.jsを作成し、以下を記述します。
const _ = require("lodash");
const request = require("request");
const fileFromBuffer = require("./fileFromBuffer");
const { getMediaUrlsFromFieldData } = require("../utils/fieldUtils");
const fetchFiles = url =>
new Promise((resolve, reject) => {
request({ url, method: "GET", encoding: null }, async (err, res, body) => {
if (err) {
reject(err);
}
const mimeType = res.headers["content-type"].split(";").shift();
const parsed = new URL(url);
const extension = parsed.pathname
.split(".")
.pop()
.toLowerCase();
resolve(fileFromBuffer(mimeType, extension, body));
});
});
const storeFiles = async file => {
const uploadProviderConfig = await strapi
.store({
environment: strapi.config.environment,
type: "plugin",
name: "upload"
})
.get({ key: "provider" });
return await strapi.plugins["upload"].services["upload"].upload(
[file],
uploadProviderConfig
);
};
const relateFileToContent = ({
contentType,
contentId,
targetField,
fileBuffer
}) => {
fileBuffer.related = [
{
refId: contentId,
ref: contentType,
source: "content-manager",
field: targetField
}
];
return fileBuffer;
};
const importMediaFiles = async (savedContent, sourceItem, importConfig) => {
const { fieldMapping, contentType } = importConfig;
const uploadedFileDescriptors = _.mapValues(
fieldMapping,
async (mapping, sourceField) => {
if (mapping.importMediaToField) {
const urls = getMediaUrlsFromFieldData(sourceItem[sourceField]);
const fetchPromises = _.uniq(urls).map(fetchFiles);
const fileBuffers = await Promise.all(fetchPromises);
const relatedContents = fileBuffers.map(fileBuffer =>
relateFileToContent({
contentType,
contentId: savedContent.id,
targetField: mapping.importMediaToField,
fileBuffer
})
);
const storePromises = relatedContents.map(storeFiles);
const storedFiles = await Promise.all(storePromises);
console.log(_.flatten(storedFiles));
return storedFiles;
}
}
);
return await Promise.all(_.values(uploadedFileDescriptors));
};
module.exports = importMediaFiles;
import-content/services/import-content.js
を変更します。
"use strict";
/** * ImportContent.js service
* * @description: A set of functions similar to controller's actions to avoid code duplication. */
const { resolveDataFromRequest, getItemsFromData } = require("./utils/utils");
const analyzer = require("./utils/analyzer");
const _ = require("lodash"); // 追加
const importFields = require("./utils/importFields"); // 追加
const importMediaFiles = require("./utils/importMediaFiles"); // 追加
const import_queue = {}; // 追加
const importNextItem = async importConfig => { // 追加
const sourceItem = import_queue[importConfig.id].shift();
if (!sourceItem) {
console.log("import complete");
await strapi
.query("importconfig", "import-content")
.update({ id: importConfig.id }, { ongoing: false });
return;
}
try {
const importedItem = await importFields(
sourceItem,
importConfig.fieldMapping
);
const savedContent = await strapi
.query(importConfig.contentType)
.create(importedItem);
const uploadedFiles = await importMediaFiles(
savedContent,
sourceItem,
importConfig
);
const fileIds = _.map(_.flatten(uploadedFiles), "id");
await strapi.query("importeditem", "import-content").create({
importconfig: importConfig.id,
ContentId: savedContent.id,
ContentType: importConfig.contentType,
importedFiles: { fileIds }
});
} catch (e) {
console.log(e);
}
const { IMPORT_THROTTLE } = strapi.plugins["import-content"].config;
setTimeout(() => importNextItem(importConfig), IMPORT_THROTTLE);
};
module.exports = {
preAnalyzeImportFile: async ctx => {
const { dataType, body, options } = await resolveDataFromRequest(ctx);
const { sourceType, items } = await getItemsFromData({
dataType,
body,
options
});
const analysis = analyzer.analyze(sourceType, items);
return { sourceType, ...analysis };
},
importItems: (importConfig, ctx) => // 追加
new Promise(async (resolve, reject) => {
const { dataType, body } = await resolveDataFromRequest(ctx);
console.log("importitems", importConfig);
try {
const { items } = await getItemsFromData({
dataType,
body,
options: importConfig.options
});
import_queue[importConfig.id] = items;
} catch (error) {
reject(error);
}
resolve({
status: "import started",
importConfigId: importConfig.id
});
importNextItem(importConfig);
}),
};
サービスの準備ができたので、controllers
に以下の関数を追加します。
create: async ctx => {
const services = strapi.plugins["import-content"].services;
const importConfig = ctx.request.body;
importConfig.ongoing = true;
const record = await strapi
.query("importconfig", "import-content")
.create(importConfig);
console.log("create", record);
await services["import-content"].importItems(record, ctx);
ctx.send(record);
}
次に、plugins/import-content/config/routes.json
に以下を追記します。
{
"method": "POST",
"path": "/",
"handler": "import-content.create",
"config": {
"policies": []
}
}
ここまでができたら、UI上からCSVファイルを選択して、Anlyze、紐付けをしてRun the Import
ボタンを押してみてください。コレクションタイプに移動すると、インポートができているはずです。
終わりに
前回の記事と併せて、CSVファイルからデータをインポートする部分を紹介しました。
公式ブログの記事には、詳細な解説もついている他、インポートの取り消し機能やインポート履歴の実装方法も紹介せされていますので、チェックしてみてください。