古典的なLZSSに可変長符号を搭載して圧縮率を稼ぐ方法を紹介します。辞書の大きさは2MB、最長一致は4168に設定。
位置情報は(A)4bits + (B)extra bits
で符号化。A=0ならBは5bits、0<A<16
ならBは6-20bits。
一致長は3~23bitsで符号化。flagはgamma符号で連長圧縮。
最長一致系列は全ての位置で求めます。最短経路問題を解くような感じで最適解を選択していきます。当然圧縮速度は控え目になります。
伸長はべらぼうに超高速で、1MB程度ならほぼ一瞬で終わります(環境次第)。
実装
// log2 bits
function l2b(a,b,c){
c=(65535<a)<<4;c|=b=(255<(a>>>=c))<<3;c|=b=(15<(a>>=b))<<2;
return((c|=b=(3<(a>>=b))<<1)|a>>b>>1)+1
}
/*compress
@In: input(Array / Uint8Array)
@done: call back of last process
done(A,a,z)
@A: compressed array of input
@a: input size
@z: compressed size
@rate: call back of progress
rate(a,z)
@a: current position
@z: last position
@return:
call with await: compressed array of @In
without await: Promise object
*/
async function compress(In,done,rate){
function puts(n,b){
for(data|=b<<bits,bits+=n;bits>7;bits-=8)Out[o++]=data&255,data>>>=8
}
const z=In.length,WB=21,W=1<<Math.min(WB,l2b(z)),M=W-1,SB=4,SN=1<<SB,MIN=3,MAX=4168,
H=new Int32Array(1<<18).fill(-1),LM=new Float64Array(z+1),
Out=[0],fn=b=>setTimeout(c=>b(rate(a,z)),0),x=rate?32767:-1;
var a=0,e=MAX-MIN,n=0,o=e<z?e:z,p=z>>15?32768:z,data,bits,
Lc=new Uint8Array(o+MIN),Dc=new Uint8Array(p);
// set length cost
for(;o;)Lc[--o+MIN]=o<4?4:o<8?5:o<12?6:o<20?8:o<71?11:l2b(o-70)+10;
// set distance cost
for(let b=WB-SN;o<p;Dc[o++]=b>WB-SN?b+SB:WB-SN+SB+1)o>>b+1&&b++;
// Pass 1: Find all matches
for(let N=new Int32Array(W*2);a<z;a&x||await new Promise(fn)){
let v=2,m=z-a;bits=Out[a];
if(m>2)scan:{
let b=0,c=0,i=0,r=a&M,l=r|W,w=a<W?-1:a-W,
s=H[p=(In[a]|In[a+1]<<8|In[a+2]<<16)*2097143>>>14];
if(m>MAX)m=MAX;
for(H[p]=a;s>w;i=b<c?b:c){
if(In[s+i]===In[a+i]){
for(;i++<m&&In[s+i]===In[a+i];);
if(i>v){// longest match
let d=~s+a,e;
if(d<32768)p=bits+Dc[d];
else{// cost is not defined
for(p=14;d>=1<<++p;);p+=bits+SB-1
}
// update cost and matches
for(d*=MAX;i>v;e>=Out[a+v]||(Out[a+v]=e,LM[a+v]=v+d))e=p+Lc[++v];
if(i>m){
N[r]=N[s];N[l]=N[s&M|W];
break scan
}
}
}
if(In[s+i]<In[a+i])N[r]=s,s=N[r=s&M|W],c=i;
else N[l]=s,s=N[l=s&M],b=i
}
N[l]=N[r]=-1
}
bits+=9;bits>=Out[++a]||(Out[a]=bits,LM[a]=In[a-1]*MAX+1)
}
// Pass 2: save best matches
for(p=a;a-=(LM[p--]=LM[a])%MAX;);
// Pass 3: Output the codes
for(Out.length=bits=e=o=0;p<z;){
let l=LM[++p],i=l/MAX|0,b,c,d,r,s;
if((l%=MAX)<MIN){
if(e){
r=l2b(s=e/2);
puts(r+r-1,1<<r-1|(s^1<<r-1)<<r);// write flags
for(s=e;e;puts(d&31,d>>>5))d=H[s-e--]// write matches
}
++a;++n;continue
}
if(a){
puts(r=l2b(a),1<<--r);puts(r,a^1<<r);// write flags
while(a)puts(8,In[n-a--])// write literals
}
d=SB;n+=l;l-=MIN;
// length code
if(l<4)b=3,c=l<<1|1;
else if(l<8)b=4,c=l-4<<2|2;
else if(l<12)b=5,c=l-8<<3|4;
else if(l<20)b=7,c=l-12<<4|8;
else if(l<71)b=10,c=l-20<<4;
else s=l2b(l-=70)-1,b=s+10,c=l-(1<<s)<<b-s|52+s<<4;
// distance code
for(r=WB-SN;i>>++r;);
if(s=--r-WB+SN)s|=i-(1<<r)<<d,d+=r;
else s|=i<<d,d+=WB-SN+1;
H[e++]=c<<5|b,H[e++]=s<<5|d
}
if(e){// last matches
let s=e/2,r=l2b(s),d;
puts(r+r-1,1<<r-1|(s^1<<r-1)<<r);
for(s=e;e;puts(d&31,d>>>5))d=H[s-e--]
}
if(a){// last literals
let r=l2b(a);
puts(r,1<<--r);puts(r,a^1<<r);
while(a)puts(8,In[n-a--])
}
puts(7,0);done&&done(Out,z,o);
return Out
}
/*decompress
@In: input(Array / Uint8Array)
@done: call back of last process
done(A,a,z)
@A: decompressed array of @In
@a: last position
@z: decompressed size
@rate: call back of progress
rate(a,z)
@a: current position
@z: last position
@return:
call with await: decompressed array of @In
without await: Promise object
*/
async function decompress(In,done,rate){
function gets(l,n){//bit reader
for(;l>bits;bits+=8)data|=In[a++]<<bits;
n=data&(1<<l)-1;data>>>=l;bits-=l;
return n
}
const size=In.length,Out=[],WB=21,SB=4,SN=1<<SB,x=rate?4095:-1,fn=b=>setTimeout(c=>b(rate(a,size)),0);
var a=0,bits=0,data,f,l,o=0,p,r=1;
a:for(;;o&x||await new Promise(fn)){
if(!--r){// update flag
for(f^=1;!gets(1);++r)if(a>size)break a;
r=1<<r|gets(r)
}
if(f){// literal
p=gets(8);if(a>size)break;
Out[o++]=p
}else{// copy
l=gets(1)?gets(2)+3:gets(1)?gets(2)+7:gets(1)?gets(2)+11:gets(1)?gets(3)+15:(l=gets(6)+23)<75?l:(gets(l-=75)|1<<l)+73;
p=gets(SB)+WB-SN;
p=~(p>WB-SN?gets(p)|1<<p:gets(p+1))+o;
while(l--)Out[o++]=Out[p++]
}
}done&&done(Out,size,o);
return Out
}
試し斬り
(async()=>{
let A=Array.from("That that is is that that is not is not is that it it is",a=>a.charCodeAt());
let rate=(a,b)=>{console.log((a/b*1e4|0)/100,"%")}, done=(a,b,c)=>{console.log(b,"->",c,a)};
let e=await compress(A,done,rate), d=await decompress(e,done,rate);
console.log(e.length, String.fromCharCode(...d))
})()
codepen召喚
See the Pen LZSS variant by xezz (@xezz) on CodePen.
benchmark
js file
file名 | 元尺 | 圧縮後 |
---|---|---|
angular-1.4.0.min.js | 144729 | 52398 |
bootstrap-3.3.6.min.js | 36868 | 10499 |
jquery-3.7.1.min.js | 87526 | 31856 |
react-0.13.3.js | 600572 | 116281 |
vue-2.7.16.js | 434871 | 93682 |
C++
C++版の要望が殺到しまくった訳が無いので、晒しておきます。JavaScript版より機能豊富で、速度調整等が可能。flagの圧縮は未実装。互換性はありません。
#ifdef __GNUC__
#define _FILE_OFFSET_BITS 64
#define _fseeki64 fseeko64
#define _ftelli64 ftello64
#define __min(a, b) ((a)<(b)?(a):(b))
#define __max(a, b) ((a)>(b)?(a):(b))
#endif // __GNUC__
#define _CRT_SECURE_CPP_OVERLOAD_STANDARD_NAMES 1
#define _CRT_SECURE_NO_WARNINGS
#define _CRT_DISABLE_PERFCRIT_LOCKS
#include<stdio.h>
#include<stdlib.h>
#include<string.h>
#include<time.h>
typedef unsigned char U8;
typedef unsigned short U16;
typedef unsigned int U32;
typedef unsigned long long U64;
const char noMem[]="not enough memory";
const int WBITS=21, // Window size (17..23)
WSIZE=1<<WBITS,
WMASK=WSIZE-1,
SLOT_BITS=4,
NUM_SLOTS=1<<SLOT_BITS,
A_BITS=2, // 1 xx
B_BITS=2, // 01 xx
C_BITS=2, // 001 xx
D_BITS=3, // 0001 xxx
E_BITS=6, // 0000 xxxxxx
M_BITS=14,
A=1<<A_BITS,
B=(1<<B_BITS)+A,
C=(1<<C_BITS)+B,
D=(1<<D_BITS)+C,
E=(1<<E_BITS)+D-M_BITS-1,
MINML=3, MAXML=E-2+MINML+(1<<M_BITS), FAR=1<<20,
BUF_SIZE=1<<26;
#define HASH_LOG 18
#define HASH_SIZE (1<<HASH_LOG)
#define NIL (-1)
FILE *ip, *op;
//////// for filter ////////
U8 isBPE[8192];
static inline void setBPE(U8 a,U8 b){
int ab=a;
ab<<=5;
isBPE[ab+=b>>3]|=(1<<(b&7));
}
static inline void unsetBPE(U8 a,U8 b){
int ab=a;
ab<<=5;
isBPE[ab+=b>>3]&=~(1<<(b&7));
}
static inline int hasBPE(U8 a,U8 b){
int ab=a;
ab<<=5;
return isBPE[ab+=b>>3]&(1<<(b&7));
}
#define BPE 1024
int bpeLastOfs[BPE];
int bpeNum, bpeHead;
void bpe_init(){
bpeNum=bpeHead=0;
memset(isBPE,0,8192);
}
void bpePush(U8 *buf, int pos){
if(pos<2)return;
U8 a=buf[pos-2], b=buf[pos-1];
if(hasBPE(a,b))return;
if(bpeNum==BPE){
int prev_pos=bpeLastOfs[bpeHead];
U8 pa=buf[prev_pos], pb=buf[prev_pos+1];
unsetBPE(pa,pb);
}
bpeLastOfs[bpeHead++]=pos-2;
if(bpeHead==BPE) bpeHead=0;
if(bpeNum<BPE) bpeNum++;
setBPE(a,b);
}
int cntBPEs(U8 *buf, int n){
int i=1, cnt=buf[1]==buf[0]&&buf[2]==buf[1];
bpe_init();
for(n--;++i<n;){
bpePush(buf,i);
if(buf[i]==buf[i-1]&&buf[i]==buf[i+1]||hasBPE(buf[i],buf[i+1]))
cnt++;
}
return cnt;
}
//////// for lzss ////////
inline void pbits(int n, U64 x){
static U64 bN=0;
static U8 bn=0;
bN|=x<<bn;
for(bn+=n;bn>7;bn-=8) putc(bN,op), bN>>=8;
}
inline U64 gbits(int n){
static U64 bN=0;
static U8 bn=0;
for(;bn<n;bn+=8) bN|=getc(ip)<<bn;
const U64 x=bN&((1<<n)-1);
bN>>=n;bn-=n;
return x;
}
inline int l2b(U32 x){
static const U8 log2[256]={0,1,2,2,3,3,3,3,4,4,4,4,4,4,4,4,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,5,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,6,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,7,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8,8};
int l=-1;
if(x>255){l+=8,x>>=8;
if(x>255){l+=8,x>>=8;
if(x>255){l+=8,x>>=8;}}}
return l+log2[x];
}
template<bool FWD>
void e89transform(U8 *B, int n){
const int end=n-8;
int p=0;
while(reinterpret_cast<int&>(B[p])!=0x4550&&++p<end);
while(p<end)if((B[p++]&254)==0xe8){
int& addr=reinterpret_cast<int&>(B[p]);
if(FWD){
if(addr>=-p&&addr<n-p) addr+=p;
else if(addr>0&&addr<n) addr-=n;
}
else if(addr<0){
if(addr+p>=0) addr+=n;
}
else if(addr<n) addr-=p;
p+=4;
}
}
void compress(U64 fs, U32 bs, int level, int cut, int much, int et){
static int H[HASH_SIZE];
static int N[WSIZE][2];
U8 *Dc, *Lc; // costs of len & dist
{
int i=0, w=__min(WSIZE,fs), m=__min(MAXML-MINML,fs), b=WBITS-NUM_SLOTS;
// encode file size
while(fs>>i)++i;
pbits(6,i);
if(!i--)return pbits(2,0);
pbits(i,fs^1<<i);
if(fs<4){
for(pbits(4,0);fs--;)pbits(9,getc(ip)<<1);
return pbits(7,0);
}
Dc=new U8[w+m+MINML], Lc=Dc+w;
for(i=m;i;) Lc[--i+MINML]=i<A? A_BITS+2: i<B? B_BITS+3: i<C? C_BITS+4: i<D? D_BITS+5: i<E? E_BITS+5:l2b(i-E+1)+E_BITS+5;
for(;i<w;Dc[i++]=b>WBITS-NUM_SLOTS?b+SLOT_BITS:WBITS-NUM_SLOTS+SLOT_BITS+1) i>>b+1&&b++;
}
if(!(level&=65535)) level=HASH_SIZE;else level+=4;
if(!(cut&=1023)) cut=HASH_SIZE;else cut+=31;
if(!(much&=8191)) much=MAXML;else much+=256;
pbits(3,bs&=7);et&=3;
bs=1<<23+bs;if(bs>fs)bs=fs;
printf("block:%d depth:%d cut:%d skip:%d filter:%d\n",bs,level,cut,much,et);
struct DisLen{U8 d[3], l[2];} *DL=new DisLen[bs+1];
U32 *P=new U32[bs+1]; // cost
U8 *buf=new U8[bs], *O=(U8*)P;
for(int n;(n=fread(buf,1,bs,ip))>0;){
if(et){
int c=cntBPEs(buf,n), d;
e89transform<true>(buf,n);
d=cntBPEs(buf,n);
if(c<d||et<2) pbits(1,1);
else pbits(1,0), e89transform<false>(buf,n);
}else pbits(1,0);
// Pass 1: Find all matches
for(int i=HASH_SIZE;i;)H[--i]=NIL;
for(int p=n;p;)P[p--]=-1;
for(int p=P[0]=0;p<n;p&65535||printf("pos:%d / %d\r",p,n)){
int best=MINML-1, limit=n-p, s;
U32 c=P[p];
if(limit>=MINML){
if(limit>MAXML) limit=MAXML;
int *L=&N[p&WMASK][1], *R=&N[p&WMASK][0];
int l=0, r=0, depth=level, same=0;
const int s0=__max(p-WSIZE, NIL);
int *h=&H[0x6b43a9b5*(0xffffff&*(U32*)&buf[p])>>32-HASH_LOG];
s=*h;
for(*h=p;s>s0&&--depth;){
int len=__min(l,r);
if(buf[s+len]==buf[p+len]){
while(++len<limit && buf[s+len]==buf[p+len]);
if(len==best && len<16 && ++same>cut) depth-=depth>>1, same-=16; // avoid too much scan. but heart ratio.
if(len>best){
int dist=p+~s;
if(len-(dist>=FAR)>=MINML) // get matches
for(U32 c1=c+Dc[dist], c2;best<len;){
c2=c1+Lc[++best];
if(c2<P[p+best])
P[p+best]=c2,
*(U32*)&DL[p+best]=dist,
*(U16*)&(DL[p+best].l)=best;
}
if(same=len==limit){
*R=N[s&=WMASK][0];
*L=N[s][1];
goto end;
}best=len;
}
}
if(buf[s+len]<buf[p+len])
*R=s, R=&N[s&WMASK][1],
s=*R, r=len;
else
*L=s, L=&N[s&WMASK][0],
s=*L, l=len;
}
*L=*R=NIL;
end:;
}
c+=9;s=buf[p++];
if(c<P[p]) P[p]=c, *(U32*)&DL[p]=s, *(U16*)&(DL[p].l)=1;
if(best>much)for(best-=best>>1;best>E;){ // skip match
limit=300; // larger is better but slower
if(limit>--best) limit=best;
int *L=&N[p&WMASK][1], *R=&N[p&WMASK][0];
int l=0, r=0, depth=level+11>>3;
const int s0=__max(p-WSIZE, NIL);
int *h=&H[0x6b43a9b5*(0xffffff&*(U32*)&buf[p])>>32-HASH_LOG];
s=*h;
for(*h=p;s>s0&&--depth;){
int len=__min(l,r);
if(buf[s+len]==buf[p+len]){
for(++len;*(U32*)&buf[s+len]==*(U32*)&buf[p+len]&&len+4<limit;)len+=4;
while(buf[s+len]==buf[p+len]&&len<limit)++len;
if(len==limit){
*R=N[s&=WMASK][0];
*L=N[s][1];
goto tail;
}
}
if(buf[s+len]<buf[p+len])
*R=s, R=&N[s&WMASK][1],
s=*R, r=len;
else
*L=s, L=&N[s&WMASK][0],
s=*L, l=len;
}
*L=*R=NIL;
tail:
c=P[p]+9;s=buf[p++];
if(c<P[p]) P[p]=c, *(U32*)&DL[p]=s, *(U16*)&(DL[p].l)=1;
}
}
// Pass 2: save best matches
U64 o=0;
for(int p=n;p;o++){
U32 d=WMASK&*(U32*)&DL[p];
U16 l=*(U16*)&(DL[p].l), a=l>1, b=0;
if(!l) l=1, d=buf[p-1]; // maybe error
p-=l;
if(a&&(l-=MINML)>15){
O[o++]=l>>4&255, ++a;
if(l>4095) O[o++]=l>>12&255, ++a;
}
O[o++]=d&255;
if(d>255){
O[o++]=d>>8, ++b;
if(d>65535) O[o++]=d>>16, ++b;
}
O[o]=b<<2|a;
if(a) O[o]|=(l<<4)&240;
}
printf("matches + literal:%I64d\n",o);
// Pass 3: Output the codes
while(o){
U32 q=O[--o], p=O[--o];
if(q){
int l=q>>4;
q&=15;
if(q>7) p=p<<8|O[--o];
if(q>3) p=p<<8|O[--o];
q&=3;
if(q>2) l|=O[--o]<<12;
if(q>1) l|=O[--o]<<4;
l-=p>=FAR;
U64 c;
U32 b, d=SLOT_BITS, e, m=l, log=WBITS-NUM_SLOTS;
if(l<A) b=A_BITS+2, c=l<<2|3;
else if(l<B) b=B_BITS+3, c=l-A<<3|5;
else if(l<C) b=C_BITS+4, c=l-B<<4|9;
else if(l<D) b=D_BITS+5, c=l-C<<5|17;
else if(l<E) b=E_BITS+5, c=l-D<<5|1;
else e=l2b(l-=E-1), b=e+E_BITS+5, c=l-(1<<e)<<b-e|E-D+e+1<<5|1;
while(p>=(2<<log))++log;
if(e=log-WBITS+NUM_SLOTS) e|=p-(1<<log)<<d, d+=log;
else e=p<<d, d+=WBITS-NUM_SLOTS+1;
pbits(b,c), pbits(d,e);
}
else pbits(9,(U8)p<<1);
}
}pbits(7,0);
delete[]Dc;delete[]P;delete[]DL;delete[]buf;
}
void decompress(){
U64 fs=gbits(6);
if(!fs--) return;
fs=1<<fs|gbits(fs);
U32 bs=1<<23+gbits(3);
if(bs>fs) bs=fs;
U8 *buf=new U8[bs];
while(fs){
int p=0, n=fs<bs?fs:bs, et=gbits(1);
while(p<n)
if(gbits(1)){
int len;
if(gbits(1)) len=gbits(A_BITS);
else if(gbits(1)) len=gbits(B_BITS)+A;
else if(gbits(1)) len=gbits(C_BITS)+B;
else if(gbits(1)) len=gbits(D_BITS)+C;
else if((len=gbits(E_BITS)+D)>E) len=(gbits(len-=E+1)|1<<len)+E-1;
int o=gbits(SLOT_BITS);
o=o? gbits(o+=WBITS-NUM_SLOTS)|1<<o: gbits(WBITS-NUM_SLOTS+1);
len+=o>=FAR, o=~o+p;
if(o<0) fprintf(stderr, "File corrupted: o=%d %d\n",o,p), exit(1);
buf[p++]=buf[o++];
buf[p++]=buf[o++];
buf[p++]=buf[o++];
while(len--) buf[p++]=buf[o++];
}
else buf[p++]=gbits(8);
if(et)e89transform<false>(buf,p);
fwrite(buf,1,p,op);fs-=p;
}if(buf)delete[]buf;
}
int main(int i, char**v){
if(i<3)usage:printf(
"LZSS variant file compressor\n\n"
"To compress: %s c[#1,#2,#3,#4,#5] infile [outfile]\n"
"#1 [0-7] buffer size. 2^(23 + #1)\n"
"#2 [0-65535] maximum search depth(0: unlimited)\n"
"#3 [0-1023] above cutter. higher is slow(0: unlimited)\n"
"#4 [0-8191] The minimal length of skip match(0: unlimited)\n"
"#5 [0-2] e8e9 filter(0:off, 1:on, 2:auto)\n"
"outfile default name is infile.crs\n\n"
"Examples:\n"
"%s c in out\n"
"%s c,0,0,0 in out\n"
"%s c0,12 in out\n"
"%s c,,,1 in out\n\n"
"To decompress: %s d infile [outfile]\n"
"outfile default name is infile.out\n"
, *v,*v,*v,*v,*v,*v), exit(1);
char *c=v[1], d=*c, *name=v[2], *name2=i<4?(char*)malloc(strlen(name)+4): v[3];
if(!name2)puts(noMem),exit(1);
clock_t t=clock();
if(i<4)
strcpy(name2, name),
strcat(name2, d=='c'?".crs":".out");
if(ip=fopen(name2,"rb"))printf("Warning: overwrite \"%s\"\n",name2), fclose(ip);
else printf("output=%s\n", name2);
ip=fopen(name,"rb");
op=fopen(name2,"wb");
if(i<4)free(name2);
if(!ip || !op)puts("File open error at the beginning."), exit(1);
if(_fseeki64(ip,0,SEEK_END)) puts("Fseek failed"), exit(1);
U64 l=_ftelli64(ip), n;
if(l<0) puts("Ftell failed"), exit(1);
rewind(ip);
if(d=='c'){
int A[]={3,8188,96,767,2},a=i=n=0,s=sizeof(A)/sizeof(A[0]);
for(;*++c&&i<s;)
if(*c<58&&*c>47){if(a<5)n=n*10+*c-48,a++;}
else{if(a)A[i]=n;i++;a=n=0;}
if(a&&i<s)A[i]=n;
compress(l,A[0],A[1],A[2],A[3],A[4]);}
else if(d=='d')decompress();
else goto usage;
n=_ftelli64(op);
printf("size:%I64d -> %I64d (%2.2f%%)\ntime:%2.3f s\n",l,n,100.0*n/l,(double)(clock()-t)/CLOCKS_PER_SEC);
fclose(ip);fclose(op);
}