アソシエーション分析: 大規模取引ログの相関分析

Overview
This post supports the association analysis for large-scale transaction data which consists of python implemenation for table data with Pandas and PySpark and its visualization with NetworkX.

Def. Metrics

The followings are standard metrics on association analysis.

• Confidence - Conditional probability
  $$
  \text{Confidence}(X \to Y) := \frac{|X \cap Y|}{|X|} \approx P(Y|X)
  $$
• Support - Item's popularity, frequency, probability. "How popular is the item X?", "How popular is the combination of items (X, Y)?"
  $$
  \begin{align}
  \text{Support}(X) &:= \frac{|X|}{|\Omega|} \approx P(X) \\
  \text{Support}(X, Y) &:= \frac{|X \cap Y|}{|\Omega|} \approx P(X \cap Y) \\
  \text{Support}(X, Y, Z) &:= \frac{|X \cap Y \cap Z|}{|\Omega|} \approx P(X \cap Y \cap z)
  \end{align}
  $$
• Lift - Efficient conditional rules with high frequency (popularity)
  $$
  \text{Lift}(X \to Y) := \frac{\text{Confidence}(X \to Y)}{\text{Support}(X)} = \frac{ \frac{|X \cap Y|}{|X|} }{ \frac{|X|}{|\Omega|} }
  \approx \frac{P(Y|X)}{P(X)}
  $$

Case1: midle-scale trans-data

derive association rules by using mlxtendm and pandas

1) Busket data

import os
import numpy as np
import pandas as pd
import networkx as nx
from mlxtend.frequent_patterns import apriori
from mlxtend.frequent_patterns import association_rules
import matplotlib.pyplot as plt
%matplotlib inline

df = pd.read_csv('/kaggle/input/my-association-analysis/toy_association_analysis_network.csv', encoding='utf-8')
basket_df = df.fillna(0).drop(["#", "Pages"], axis=1)
basket_d

2) Assoc Rules

Run Apriori-Algorithm
Assoc Rules within Supp > 0.1, Lift > 1.0

# Apriori algorithm
freq_items1 = apriori(basket_df, min_support = 0.1, use_colnames = True)

# Sort results by support-value
freq_items1 = freq_items1.sort_values('support', ascending = False)

# Check results
display(freq_items1.head(5))
display(freq_items1.tail(5))

# Num of itemsets
print(freq_items1.shape[0])

# Extract association rules
a_rules1 = association_rules(freq_items1, metric = "lift", min_threshold = 1)

# Sort rules by lift-value
a_rules1 = a_rules1.sort_values('lift',ascending = False).reset_index(drop=True)

# Type change: frozensets -> string
a_rules1['antecedents'] = a_rules1['antecedents'].apply(lambda x: ', '.join(list(x))).astype("unicode")
a_rules1['consequents'] = a_rules1['consequents'].apply(lambda x: ', '.join(list(x))).astype("unicode")

# Check results
display(a_rules1.head(10))

# Num of rules
print(a_rules1.shape[0])

3) Visualization

Visualization of graphical model (NetworkX)

# New graph
G = nx.from_pandas_edgelist(a_rules1, 'antecedents', 'consequents', ['lift'])

# Graph layout
pos = nx.spring_layout(G)
pr = nx.pagerank(G) # pagerank

# Visualization
plt.figure(figsize=(12, 12))
nx.draw_networkx_edges(G, pos, 
                       edge_color='silver', 
                       width=0.3)
nx.draw_networkx_nodes(G, pos, 
                       node_color=list(pr.values()), 
                       node_size=[30000 * v for v in pr.values()],
                       cmap=plt.cm.Blues, 
                       edgecolors="gray",
                       alpha=0.5)

plt.axis('off')
plt.show()

# New Graph
G = nx.from_pandas_edgelist(a_rules1, 'antecedents', 'consequents', ['lift'])

# Graph layout
pos = nx.spring_layout(G)
pr = nx.pagerank(G) # pagerank

# Visualization
plt.figure(figsize=(12, 12))

nx.draw_networkx_edges(G, pos,
                       arrows=True,
                       arrowsize=30,
                       edge_color='silver', 
                       width=1.0)

nx.draw_networkx_nodes(G, pos, 
                       node_color=list(pr.values()), 
                       node_size=[40000 * v for v in pr.values()],
                       cmap=plt.cm.Blues, 
                       edgecolors="gray",
                       alpha=0.75)

# Position nodes using Fruchterman-Reingold force-directed algorithm.
nx.draw_networkx_labels(G, pos,
                       font_size=10)

plt.axis('off')
plt.show()

Case2: large-scale trans-data

derive association rules by using PySpark.

1) Busket data

import pyspark.sql.functions as fn
from pyspark.ml.feqture import VectorAssembler
from pyspark.ml.fpm import FPGrowth

def make_basket_data(spark, input_sdf, customer_id_column, items_col_name, flg_columns_list):
    for idx, flg_column in enumerate(flg_columns_list):
        temp_sdf = input_sdf.withColumn('customer_behavior', fn.when(fn.col(flg_column) == 1, fn.lit(flg_column.replace('_flg', '')))).select(customer_id_column, 'customer_behavior')

        if idx == 0:
            res_sdf = temp_sdf
        else:
            res_sdf = res_sdf.union(temp_sdf)

    basket_data = res_sdf.groupBy(customer_id_column).agg(fn.collect_list('customer_behavior').alias('customer_behavior_collection'))
    return basket_data


def fit_association_rules(spark, basket_data, items_col_name='item_collection', min_supp=0.006, min_conf=0.006):
    # make model && fit basket data
    fpGrowth = FPGrowth(itemsCol=items_col_name, minSupport=min_supp, minConfidence=min_conf)
    model = fpGrowth.fit(basket_data)
    return model

flg_columns_list = [
    'item_1_use_flg',
    'item_2_use_flg',
    'item_3_use_flg',
    ...
    'item_N_use_flg',
]

# make basket data
basket_data = make_basket_data(spakr, input_sdf, customer_id_column='customer_id', items_col_name='item_collection', flg_columns_list=flg_columns_list)
basket_data.show()

2) Assoc Rules

Run Apriori-Algorithm

# FPGrowth algorithm; find association rules
model = fit_association_rules(spark, basket_data, items_col_name='item_collection', min_supp=0.006, min_conf=0.006)

# Frequent item sets
freq_behaviorset_sdf = model.freqItemsets()
freq_itemset_sdf.show()

# Association Rules
association_rules_sdf = model.associatoinRules
association_rules_sdf.show(truncate=False)

3) Visualization

same as the above.