ルールがよくわからない。。最後の3つはなんで成功するんだろう。
library(dplyr)

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1))
# > Error: not compatible with STRSXP

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1:5))
# > Error: not compatible with STRSXP

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1, b=2))
# > Error: wrong result size (2), expected 5 or 1

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1:5, b=2:6))
# > Error: wrong result size (2), expected 5 or 1

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1, b=2, c=3, d=4, e=5))
# > Error in match.names(clabs, names(xi)) : 
# >   names do not match previous names
# > In addition: Warning message:
# > In format.data.frame(x, digits = digits, na.encode = FALSE) :
# >   corrupt data frame: columns will be truncated or padded with NAs

data.frame(X=1:5) %>% mutate(X2 = data.frame(a=1:5, b=2:6, c=3:7, d=4:8, e=5:9))
# >   X X2.a X2.b X2.c X2.d X2.e
# > 1 1    1    2    3    4    5
# > 2 2    2    3    4    5    6
# > 3 3    3    4    5    6    7
# > 4 4    4    5    6    7    8
# > 5 5    5    6    7    8    9


data.frame(X=1:5) %>% mutate(X2 = list(a=1, b=2, c=3, d=4, e=5))
# >   X X2
# > 1 1  1
# > 2 2  2
# > 3 3  3
# > 4 4  4
# > 5 5  5

data.frame(X=1:5) %>% mutate(X2 = list(a=1:5, b=2:6, c=3:7, d=4:8, e=5:9))
# >   X            X2
# > 1 1 1, 2, 3, 4, 5
# > 2 2 2, 3, 4, 5, 6
# > 3 3 3, 4, 5, 6, 7
# > 4 4 4, 5, 6, 7, 8
# > 5 5 5, 6, 7, 8, 9