概要

Confluent が提供する kafka-connect-datagen を使用して生成可能なサンプルデータを整理しました。本記事のデータは実際にツールを用いて生成したものではなく、下記リンク先の定義に基づき生成AIを使用して作成したため、一部に誤りが含まれる可能性があります。ご了承のうえご参照ください。

引用元：kafka-connect-datagen/src/main/resources at master · confluentinc/kafka-connect-datagen

本記事は、以下のシリーズの一部です。

引用元：Data in Motion 実現へ：Data Streaming Platform である Confluent の全貌 #Kafka - Qiita

サンプルデータ

campaign_finance

{
    "time": 1619273364600,
    "candidate_id": "A12345678",
    "party_affiliation": "DEM",
    "contribution": 3500
}

clickstream_codes

{
    "code": 200,
    "definition": "Successful"
}

clickstream

{
    "ip": "111.152.45.45",
    "userid": 1,
    "remote_user": "-",
    "time": "379521",
    "_time": 1609459200010,
    "request": "GET /index.html HTTP/1.1",
    "status": "200",
    "bytes": "278",
    "referrer": "-",
    "agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
}

clickstream_users

{
    "user_id": 1,
    "username": "akatz1022",
    "registered_at": 1407645330000,
    "first_name": "Elwyn",
    "last_name": "Vanyard",
    "city": "Palo Alto",
    "level": "Gold"
}

credit_cards

{
    "card_id": 1,
    "card_number": 6011601160116611,
    "cvv": 832,
    "expiration_date": "12/27"
}

device_information

{
    "device_ip": "204.149.41.63",
    "mac_address": "20-FC-45-56-D4-11",
    "owner": "Frieda Baldi"
}

fleet_mgmt_description

{
    "vehicle_id": 1234,
    "driver_name": "Frieda Baldi",
    "license_plate": "AB123456C"
}

fleet_mgmt_location

{
    "vehicle_id": 1234,
    "location": {
        "latitude": 37.416834,
        "longitude": -121.975002
    },
    "ts": 1609459200000
}

fleet_mgmt_sensors

{
    "vehicle_id": 1234,
    "engine_temperature": 200,
    "average_rpm": 3000
}

gaming_games

{
    "id": 1000,
    "room_name": "Arcade -- Rookie",
    "created_date": 1609459200000
}

gaming_player_activity

{
    "player_id": 1001,
    "game_room_id": 1000,
    "points": 250,
    "coordinates": "[12,34]"
}

gaming_players

{
    "player_id": 1001,
    "player_name": "Frieda Baldi",
    "ip": "117.119.34.13"
}

insurance_customer_activity

{
    "activity_id": 1,
    "customer_id": 415,
    "activity_type": "web_open",
    "propensity_to_churn": 0.25,
    "ip_address": "223.135.0.204"
}

insurance_customers

{
    "customer_id": 415,
    "first_name": "Herold",
    "last_name": "Agdahl",
    "email": "hagdahlbi@last.fm",
    "gender": "Male",
    "income": 444152,
    "fico": 423,
    "years_active": 1
}

insurance_offers

{
    "offer_id": 1,
    "offer_name": "new_home_policy",
    "offer_url": "http://privacy.gov.au/in/faucibus/orci/luctus.js"
}

inventory

{
    "id": 0,
    "quantity": 0,
    "productid": 0
}

orders

{
    "ordertime": 1519273364600,
    "orderid": 1,
    "itemid": "Item_1",
    "orderunits": 1.0,
    "address": {
        "city": "City_1",
        "state": "State_1",
        "zipcode": 10001
    }
}

pageviews

{
    "viewtime": 1694763296000,
    "userid": "User_1",
    "pageid": "Page_10"
}

payroll_bonus

{
    "employee_id": 1000,
    "bonus": 10,
    "ts": 1609459200000
}

payroll_employee

{
    "employee_id": 1000,
    "first_name": "Carleton",
    "last_name": "Beagan",
    "age": 32,
    "ssn": "723-37-6885",
    "hourly_rate": 20,
    "gender": "female",
    "email": "xcqn@mycompany.com"
}

payroll_employee_location

{
    "employee_id": 1000,
    "lab": "lab-1",
    "department_id": 5,
    "arrival_date": 18001
}

pizza_orders

{
    "store_id": 1,
    "store_order_id": 1000,
    "coupon_code": 1001,
    "date": 18000,
    "status": "accepted",
    "order_lines": [
        {
            "product_id": 66,
            "category": "calzone",
            "quantity": 4,
            "unit_price": 18.78,
            "net_price": 75.12
        },
        {
            "product_id": 71,
            "category": "salad",
            "quantity": 4,
            "unit_price": 4.90,
            "net_price": 19.60
        }
    ]
}

pizza_orders_cancelled

{
    "store_id": 1,
    "store_order_id": 1001,
    "date": 18000,
    "status": "cancelled"
}

pizza_orders_completed

{
    "store_id": 1,
    "store_order_id": 1000,
    "date": 18000,
    "status": "completed",
    "rack_time_secs": 150,
    "order_delivery_time_secs": 1200
}

product

{
    "id": 0,
    "name": "Sample Product",
    "description": "This is a sample product description.",
    "price": 19.99
}

purchase

{
    "id": 0,
    "item_type": "boots",
    "quantity": 1,
    "price_per_unit": 10.00
}

ratings

{
    "rating_id": 1,
    "user_id": 10,
    "stars": 4,
    "route_id": 123,
    "rating_time": 1694763296000,
    "channel": "mobile"
}

shoe_clickstream

{
    "stream_id": "abc123",
    "product_id": "shoe_001",
    "user_id": "user_456",
    "view_time": 120,
    "page_url": "http://example.com/shoe_001",
    "ip": "192.168.1.1",
    "ts": 1694763296000
}

shoe_customers

{
    "id": "01a429f8-f576-4d9b-8d0e-4d5e8ef37e09",
    "first_name": "Cyrill",
    "last_name": "Haeslier",
    "email": "vbiaggelli0@prweb.com",
    "phone": "615-978-0286",
    "street_address": "7 Hooker Court",
    "state": "Massachusetts",
    "zip_code": "2119",
    "country": "United States",
    "country_code": "US"
}

shoe_orders

{
    "order_id": "1000",
    "product_id": "828044c8-832b-48b4-b1f0-89a500496335",
    "customer_id": "01a429f8-f576-4d9b-8d0e-4d5e8ef37e09",
    "ts": "1609459200000"
}

shoes

{
    "sale_price": 7495,
    "id": "828044c8-832b-48b4-b1f0-89a500496335",
    "brand": "Jones-Stokes",
    "name": "TrailRunner Impreza 239",
    "rating": 0.0
}

siem_logs

{
    "log_id": "log_001",
    "timestamp": 1694763296000,
    "source_ip": "192.168.1.1",
    "destination_ip": "192.168.1.2",
    "event_type": "login_attempt",
    "status": "success",
    "user": "jdoe"
}

siem_logs

{
    "hostname": "204.149.41.63",
    "action": "allow",
    "l4": "tcp",
    "access_group": "group-1",
    "source": {
        "ip": "54.222.176.193",
        "port": 10123
    },
    "destination": {
        "ip": "122.120.109.66",
        "port": 10567
    }
}

siem_logs

{
    "hostname": "204.149.41.63",
    "action": "allow",
    "l4": "tcp",
    "access_group": "group-1",
    "source": {
        "ip": "54.222.176.193",
        "port": 10123
    },
    "destination": {
        "ip": "122.120.109.66",
        "port": 10567
    }
}

StockTrade

{
    "side": "SELL",
    "quantity": 3200,
    "symbol": "ZXZZT",
    "price": 450,
    "account": "LMN456",
    "userid": "User_7"
}

stores

{
    "store_id": 1,
    "city": "Raleigh",
    "state": "NC"
}

syslog_logs

{
    "name": "LogName",
    "type": "RFC5424",
    "message": "system update",
    "host": "121.46.66.201",
    "version": "3.25.1",
    "tag": ".source.s_src",
    "level": 123,
    "facility": "syslog",
    "severity": 5,
    "appName": "APPLOG",
    "remoteAddress": "91.253.222.9",
    "rawMessage": "update complete",
    "processId": "1234567",
    "messageId": "123",
    "deviceVendor": "loggy",
    "deviceProduct": "syspr",
    "deviceVersion": "1.0",
    "ts": 1609459200100
}

transactions

{
    "transaction_id": 1,
    "card_id": 12,
    "user_id": "User_3",
    "purchase_id": 0,
    "store_id": 5
}

users_array_map

{
    "registertime": 1500000000000,
    "userid": "User_3",
    "regionid": "Region_4",
    "gender": "FEMALE",
    "interests": ["News", "Travel"],
    "contactinfo": {
        "phone": "6503349999",
        "city": "San Mateo",
        "state": "CA",
        "zipcode": "94403"
    }
}

users

{
    "registertime": 1510000000000,
    "userid": "User_5",
    "regionid": "Region_3",
    "gender": "MALE"
}