0
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?

@mashuel(恵瑠 真珠)

JavaScript: 文字Code自作講座

0
Last updated at Posted at 2024-07-23

Shift-JISとかUnicodeとかLatin1とかいう類の文字Codeの話です。できるだけ多くの文字を表現でき、bit数も少ない独自形式を作っていきます。

Escaped UTF16(EU16)

escape文字でUnicode区間を切り替えていく。復号が簡単で快速。符号の種類が少なく、Unicode番号が近い文字が密集していれば符号文は短くなる。

Escaped UTF16a(EU16a)

Shift JISで扱う文字は殆ど1～2 Bytesで表現可能。JIS漢字も殆ど2Bytesで扱える。上記EU16も復号可能。

Shift JIS + UTF16(SJU)

ASCII文字は1 Byte、いわゆるJIS漢字は殆ど2Bytesで表現し、Shift JISっぽい符号になる。それ以外は3 Bytesで符号化。

実装編

念の為UTF8のようなものも実装。2025/10/18 Shift JIS実装

if(!this.CharCoder){
//Shift JIS用の文字変換表
const toSJISmap=(b7,hk,ic)=>{
	let a=160,S={__proto__:null},D=new TextDecoder('sjis');
	for(let b,s,V=new Uint8Array(2);++a<221;)
		for(V[0]=a^32,b=63;b<252;)if(++b^127)
			V[1]=b,s=D.decode(V),
			s.charCodeAt()^65533&&(S[s]=V[1]|V[0]<<8);
	D=String.fromCharCode;
	if(b7)for(a=128;S[D(--a)]=a;);
	if(hk)for(a=63;a;)S[D(--a+0xff61)]=a+161;
	if(ic)S["\x80"]=128,S["\uF8F0"]=160,S["\uF8F1"]=253,S["\uF8F2"]=254,S["\uF8F3"]=255;//不正文字
	return S
};
this.CharCoder={
// Shift JIS
eSJIS(S){
	var a=0,b=0,c,z=S.length,m=this.sjism||(this.sjism=toSJISmap(1,1)),A=[];
	for(;a<z;A[b++]=c)c=m[S.charAt(a++)]??33093,c>>8&&(A[b++]=c>>8,c&=255);
	return A
},
dSJIS(A){return new TextDecoder("sjis").decode(A.buffer||new Uint8Array(A))},
// EU16
eEU16(S){
	for(var a=0,b=0,c,d,k=128,n,z=S.length,A=[];a<z;A[b++]=d){
		c=S.charCodeAt(a++);
		if(d=c&127,n=c>>7,c<256){
			if(k!=(n|=128))k=A[b++]=n
		}else if(c<0x3100&&c>0x2FFF){
			if(k!=(n+=34))k=A[b++]=n
		}else if(c>0xFEFF){
			if(k!=(n-=378))k=A[b++]=n
		}else if(c<0x2680&&c>0x20FF){
			if(k!=(n+=70))k=A[b++]=n
		}else if(c>0x047F||c<0x0380){
			if(k!=(n=(c-(c%=19968))/19968+147))k=A[b++]=n;
			A[b++]=(c-(d=c%156))/156
		}else if(k!=(n+=127))k=A[b++]=n
	}return A
},
dEU16(A){
	var a=0,b=0,c,d=0,e=1,z=A.length,B=[];
	for(;a<z;c<128?B[b++]=d+(e?c:c*156+A[a++]):d=(c&=e=31)<2?c<<7:c<4?c+94<<7:c>18?(e=0,c-19)*19968:c<6?c+506<<7:c<8?c+1<<7:c+58<<7)c=A[a++];
	B.length=b;return CharCoder.by(B)
},
// EU16a
eEU16A(S){
	for(var a=0,b=0,c,d,k=128,n,z=S.length,A=[];a<z;A[b++]=d){
		c=S.charCodeAt(a++);d=c&127;n=c>>7;
		if(c<256){
			if(k!=(n|=128))k=A[b++]=n
		}else if(c<0x3100&&c>0x2FFF){
			if(k!=(n+=34))k=A[b++]=n
		}else if(c>0x4DFF&&c<0xA010)
			c-=19968,A[b++]=((c-(c%=208))/208)+151,d=c+32;
		else if(c>0xFEFF){
			if(k!=(n-=378))k=A[b++]=n
		}else if(c<0x2680&&c>0x20FF){
			if(k!=(n+=70))k=A[b++]=n
		}else if(c>0x047F||c<0x0380){
			if(k!=(n=(c-(c%=19968))/19968+147))k=A[b++]=n;
			A[b++]=(c-(d=c%156))/156
		}else if(k!=(n+=127))k=A[b++]=n
	}return A
},
dEU16A(A){
	var a=0,b=0,c,d=0,e=1,z=A.length,B=[];
	for(;a<z;c<128?B[b++]=d+(e?c:c*156+A[a++]):c>150?B[b++]=(c-151)*208+A[a++]+19936:d=(c&=e=31)<2?c<<7:c<4?c+94<<7:c>18?(e=0,c-19)*19968:c<6?c+506<<7:c<8?c+1<<7:c+58<<7)c=A[a++];
	B.length=b;return CharCoder.by(B)
},
// SJU
eSJU(S,n){
	if((n&=255)<32||n>64)n=64;
	var a=0,b=0,c,m=252-n,p=127-n,z=S.length,A=[];
	for(;a<z;A[b++]=c&255)
		if((c=S.charCodeAt(a++))>127)
			if(c<0x3100&&c>0x2FFF)A[b++]=128;
			else if(c>0x4DFF&&c<0xA040)A[b++]=(((c-=19968)-(c%=m))/m)+129,c+=c<p?n:n+1;
			else if(c>0xFEFF)A[b++]=241;
			else if(c>0x1FFF&&c<0x2700)A[b++]=(c>>8)+210;
			else if(c<0x500&&c>0x2FF)A[b++]=(c>>8)+249;
			else A[b++]=251,A[b++]=c>>8;
	return A
},
dSJU(A,n){
	if((n&=255)<32||n>64)n=64;
	var a=0,b=0,c,m=252-n,z=A.length,B=[];
	for(;a<z;)c=A[a++],
		B[b++]=c<128?c
		:c<129?12288|A[a++]
		:c<241?(c-129)*m+(c=A[a++])-(c<127?n:n+1)+19968
		:c<242?65280|A[a++]
		:c<249?c-210<<8|A[a++]
		:c<251?c-249<<8|A[a++]
		:A[a++]<<8|A[a++];
	B.length=b;
	return CharCoder.by(B)
},
// UTF8
eU8(S){// utf8 encoding
	if(self.TextEncoder)return new TextEncoder().encode(S);
	var a=0,b,A=new Uint8Array(S.length*4);
	for(b of S){
		b=b.codePointAt();
		if(b>127){
			if(b<2048)A[a++]=b>>6|192;
			else{
				if(b<65536)A[a++]=b>>12|224;
				else A[a++]=b>>18|240,A[a++]=b>>12&63|128;
				A[a++]=b>>6&63|128
			}b=b&63|128
		}A[a++]=b
	}return new Uint8Array(A.buffer,0,a)
},
dU8(A){// utf8 decoding
	if(self.TextDecoder)return new TextDecoder().decode(A);
	var a=0,b=0,c,z=A.length,B=[],S=String.fromCodePoint;
	if(S)for(;a<z;B[b++]=c<128?c:c<224?(c&31)<<6|A[a++]&63:c<240?(c&15)<<12|(A[a++]&63)<<6|A[a++]&63:(c&7)<<18|(A[a++]&63)<<12|(A[a++]&63)<<6|A[a++]&63)
			c=A[a++];
	else for(S=String.fromCharCode;a<z;)
		if((c=A[a++])<240)
			B[b++]=c<128?c:c<224?(c&31)<<6|A[a++]&63:(c&15)<<12|(A[a++]&63)<<6|A[a++]&63;
		else c=((c&7)<<18|(A[a++]&63)<<12|(A[a++]&63)<<6|A[a++]&63)-65536,
			B[b++]=0xd800|c>>10,B[b++]=0xdc00|c&0x3ff;
	if(z<65535)return S.apply(0,B);
	for(a=0,z="";a<b;)z+=S.apply(0,B.slice(a,a+=65536));
	return z
},
by(A){
	let a=A.length,S=String.fromCharCode;
	if(a<65536)S.apply(0,A);
	let b=0,B="";
	for(;b<a;)B+=S.apply(0,A.slice(b,b+=65536));
	return B
}}};
使用例
var s="あいアイαβgammma delta一丁";
var e=CharCoder.eEU16(s), d=CharCoder.dEU16(e);
console.log(e,d)

試し斬り

See the Pen Unicode converter by xezz (@xezz) on CodePen.

benchmark

LZMAによる源氏物語 1巻 桐壷の圧縮結果

符号化方式 符号文 圧縮後
Shift-JIS 32963 14924
SJU 32963 15245
EU16 30393 15316
EU16a 25630 15220
UTF8 48739 16136

Shift-JIS様の圧縮しやすさには脱帽。しかしJavaScriptでShift-JISを模倣するには面倒臭い配列を用意しなければならず、実用的ではないのだ。符号文はEU16aがぶっちぎりで小さい

0
0
0

Register as a new user and use Qiita more conveniently

  1. You get articles that match your needs
  2. You can efficiently read back useful information
  3. You can use dark theme
What you can do with signing up
Sign upLogin
0
0

Delete article

Deleted articles cannot be recovered.

Draft of this article would be also deleted.

Are you sure you want to delete this article?