ゲーム翻訳ツール

　 　 サンプルコード

GAS

function doPost(e) {
	var a = [e.postData.getDataAsString(),'en','ja']
	return ContentService.createTextOutput(LanguageApp.translate(...a))
}

MainWindow.xaml.cs

namespace Konjac{
	public partial class MainWindow : Window{
		public MainWindow(){
			InitializeComponent();
			webView.NavigationCompleted += (s, e) => {
				webView.CoreWebView2.AddHostObjectToScript("bridge", new Bridge(webView));
			};
			webView.Source = new Uri(GetPath() + "\\Konjac.html");
		}
		private static string GetPath(){
			var p = System.Diagnostics.Process.GetCurrentProcess();
			return System.IO.Path.GetDirectoryName(p.MainModule.FileName);
		}
	}
}

Bridge.cs

public class Native{
	// DPI (PerMonitorV2)
	//https://msdn.microsoft.com/en-us/library/windows/desktop/dd145062(v=vs.85).aspx
	//https://msdn.microsoft.com/en-us/library/windows/desktop/dn280510(v=vs.85).aspx
	public const int WM_DPICHANGED = 0x02E0;
	[DllImport("User32.dll")]
	public static extern IntPtr MonitorFromPoint([In] Point pt, [In] uint dwFlags);
	[DllImport("Shcore.dll")]
	public static extern IntPtr GetDpiForMonitor([In] IntPtr hmonitor, [In] DpiType dpiType, [Out] out uint dpiX, [Out] out uint dpiY);
	public enum DpiType{
		Effective = 0,
		Angular = 1,
		Raw = 2,
	}
}
public class ScreenData{
	public uint dpi { get; set; }
	public Screen screen { get; set; }
	public ScreenData(Screen s){
		var p = new Point(s.Bounds.Left + 1, s.Bounds.Top + 1);
		var m = Native.MonitorFromPoint(p, 2/*MONITOR_DEFAULTTONEAREST*/);
		uint x, y;
		Native.GetDpiForMonitor(m, Native.DpiType.Effective, out x, out y);
		screen = s;
		dpi = x;
	}
}
public class BridgeEvent{
	public string type { get; set; }
	public string value { get; set; }
	public BridgeEvent(string t, string v){
		type = t;
		value = v;
	}
	public string ToJson(){
		return JsonSerializer.Serialize(this);
	}
}

[ComVisible(true)]
public class Bridge{
	Bitmap capture;
	WebView2 view;
	public Bridge(WebView2 w){
		view = w;
	}
	public async void Scan(int x, int y, int w, int h){
		Bitmap textBmp = new Bitmap(w, h);
		Graphics g = Graphics.FromImage(textBmp);
		g.DrawImage(capture, new Point(-x, -y));
		g.Dispose();
		string file = "Konjac.png";
		string path = $"{GetPath()}\\{file}";
		textBmp.Save(path);
		string s = await OCR(file);
		Console.WriteLine(s);
		var e = new BridgeEvent("SCAN_DONE", s);

		//jsからの非同期呼び出しでnullが返る
		//https://github.com/MicrosoftEdge/WebView2Feedback/issues/822
		view.CoreWebView2.PostWebMessageAsString(e.ToJson());
	}
	public async Task<string> OCR(string file){
		//https://water2litter.net/rye/post/c_ocr_hello/

		Stream input_stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
		var source = new WriteableBitmap(System.Windows.Media.Imaging.BitmapFrame.Create(input_stream));
		input_stream.Close();

		PngBitmapEncoder encoder = new PngBitmapEncoder();
		encoder.Frames.Add(System.Windows.Media.Imaging.BitmapFrame.Create(source));
		MemoryStream temp_stream = new MemoryStream();
		encoder.Save(temp_stream);

		var converted_stream = WindowsRuntimeStreamExtensions.AsRandomAccessStream(temp_stream);

		var decorder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(converted_stream);
		SoftwareBitmap bitmap = await decorder.GetSoftwareBitmapAsync();
		converted_stream.Dispose();
		temp_stream.Close();

		OcrEngine engine = OcrEngine.TryCreateFromLanguage(new Windows.Globalization.Language("en"));
		var result = await engine.RecognizeAsync(bitmap);
		return result.Text;
	}
	public string Post(string url, string data){
		//https://docs.microsoft.com/ja-jp/dotnet/framework/network-programming/how-to-send-data-using-the-webrequest-class

		Console.WriteLine(url);
		byte[] byteArray = Encoding.UTF8.GetBytes(data);
		var request = (HttpWebRequest)WebRequest.Create(url);
		request.Method = "POST";
		request.ContentType = "application/x-www-form-urlencoded";
		request.ContentLength = byteArray.Length;
		Stream dataStream = request.GetRequestStream();
		dataStream.Write(byteArray, 0, byteArray.Length);
		dataStream.Close();

		var response = (HttpWebResponse)request.GetResponse();
		string responseContent = string.Empty;
		using (dataStream = response.GetResponseStream())
		{
			StreamReader reader = new StreamReader(dataStream);
			string responseFromServer = reader.ReadToEnd();
			Console.WriteLine(responseFromServer);
			response.Close();
			return responseFromServer;
		}
	}
	public string CaptureScreen(int i){
		Screen s = Screen.AllScreens[i];
		capture = new Bitmap(s.Bounds.Width, s.Bounds.Height);
		Graphics g = Graphics.FromImage(capture);
		g.CopyFromScreen(new Point(s.Bounds.X, s.Bounds.Y), new Point(0, 0), capture.Size);
		g.Dispose();
		string file = $"Screen{i}.png";
		string path = $"{GetPath()}\\{file}";
		Console.WriteLine(path);
		capture.Save(path);
		return file;
	}
	private static string GetPath(){
		var p = System.Diagnostics.Process.GetCurrentProcess();
		return Path.GetDirectoryName(p.MainModule.FileName);
	}
	public string getAllScreens(){
		List<ScreenData> screens = new List<ScreenData>();
		foreach (Screen s in Screen.AllScreens){
			screens.Add(new ScreenData(s));
		}
		return JsonSerializer.Serialize(screens);
	}
}

Konjac.html

<!DOCTYPE html><html lang="ja"><head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script>

DEPLOY_ID = '***************************' //GAS

$(e=>{$('body').append(new Konjac().view)})

class Konjac{
	constructor(){
		Konjac.native = new Konjac.Bridge().ready(v=>{
			v.getScreens(r=>{
				this.item = r.map((v,i)=>new Konjac.Screen(v,i))
				this.init()
			})
		})
		this.view = _html('.container')
	}
	init(){
		this.currentItem = this.item[0]
		var screens = this.item.sort((a,b)=>{return a.x-b.x})
		screens.forEach(v=>v.menuItem.click(e=>this.select(v)))
		
		this.view.append(
			_html('.content',screens.map(v=>v.view)),
			this.overlay = _html('.overlay').hide().click(e=>this.overlay.hide()),
			_html('.header',
				_html('ul',screens.map(v=>v.menuItem)),
				_html('.button,翻訳する').click(e=>{
					this.overlay.empty().show()
					this.ocr()
				})
			),
		)
		Konjac.native.hook.SCAN_DONE = e=>{
			this.overlay.append(_html('.en',e.value))
			this.translate(e.value,ja=>{
				this.overlay.prepend(_html('.ja',ja))
			})
		}
		var g = new Konjac.Gamepad({
			GAMEPAD_A:e=>this.rescan(),
			GAMEPAD_B:e=>{this.overlay.hide()}
		})
		this.select(this.currentItem)
	}
	rescan(){
		this.currentItem.capture(r=>{
			this.overlay.empty().show()
			this.ocr()
		})
	}
	select(item){
		this.overlay.hide()
		this.currentItem = item
		this.item.forEach(v=>{
			v.menuItem.removeClass('selected')
			v.view.hide()
		})
		item.menuItem.addClass('selected')
		item.view.show()
		item.capture()
	}
	ocr(){
		var dpr = this.currentItem.dpr
		var w = $(window)
		var rect = [	
			Math.round(w.scrollLeft()*dpr),
			Math.round(w.scrollTop()*dpr),
			Math.round(w.width()*dpr),
			Math.round((w.height()-$('.header').height())*dpr),
		]
		Konjac.native.ocr(...rect)
		return this
	}
	translate(data,success){
		var url = `https://script.google.com/macros/s/${DEPLOY_ID}/exec`;
		Konjac.native.post(url,data,v=>success(v))
		return this
	}
}

Konjac.Gamepad = class KonjacGamepad{
	constructor(hook){
		var s = 'A,B,X,Y,LB,RB,LT,RT,BACK,START,LS,RS,UP,DOWN,LEFT,RIGHT,HOME'
		this.types	= s.split(',').map(s=>'GAMEPAD_'+s)
		this.hook	= hook
		this.stack	= []
		this.length	= 0
		window.addEventListener("gamepadconnected",e=>{
			console.log(e)
			this.stack[e.gamepad.index] = [e.gamepad]
			this.length++
		});
		window.addEventListener("gamepaddisconnected",e=>{
			console.log(e)
			this.stack[e.gamepad.index] = []
			this.length--
		});
		setInterval(e=>this.loop(),1000/30)
	}
	loop(){
		if(this.length==0) return
		var items = [...navigator.getGamepads()].filter(Boolean)
		if(items.length==0) return
		var a = this.stack[items[0].index]
		a.unshift(items[0])
		a[0].buttons.forEach((b,i)=>{
			if(b.pressed==true) return
			if(a[1].buttons[i].pressed==false) return
			this.dispatch(this.types[i])
		})
		this.stack[items[0].index] = a.slice(0,1)
	}
	dispatch(type){
		console.log(type)
		if(this.hook[type]) this.hook[type]()
	}
}

Konjac.Screen = class KonjacScreen{
	constructor(data,i){
		var b = data.screen.Bounds
		var dpr = data.dpi/96
		this.x	= b.Left
		this.y	= b.Top
		this.w	= b.Width
		this.h	= b.Height
		this.width	= Math.round(b.Width/dpr)
		this.height	= Math.round(b.Height/dpr)
		this.dpr 	= dpr//devicePixelRatio
		this.name	= data.screen.DeviceName.replace(/[^\w\s]/g,'')
		this.index	= i
		Object.assign(this,data)
		var s = b.Width+' x '+b.Height
		this.sizeText = Konjac.Screen.type[s]||s
		this.menuItem = _html('li',this.name,' ',_html('span',this.sizeText))
		this.view = _html('img').css({width:this.width})
	}
	capture(f){
		Konjac.native.capture(this.index,r=>{
			this.view.attr('src',r+'?'+Math.random())
			if(f) f(r)
		})
		return this
	}
}
Konjac.Screen.type = {
	'1920 x 1080':'FHD',
	'1920 x 1200':'WUXGA',
	'2560 x 1440':'WQHD',
	'3840 x 2160':'4K',
}

Konjac.Bridge = class KonjacBridge{
	constructor(){
		this.hook = {}
		this.on(()=>{}).ready(()=>{})
		this.hostObjects = {bridge:{}}
		if(chrome.webview){
			this.hostObjects = chrome.webview.hostObjects
			chrome.webview.addEventListener('message',e=>{
				var data = JSON.parse(e.data)
				var hook = this.hook[data.type]||this.hook.on
				hook(data)
			})
		}
		setTimeout(e=>this.hook.ready(this),100)
	}
	on(f){this.hook.on = f
		return this
	}
	ready(f){this.hook.ready = f
		return this
	}
	getScreens(f){
		this.hostObjects.bridge.getAllScreens().then(r=>{
			f(JSON.parse(r))
		}).catch(console.log)
		return this
	}
	capture(i,f){
		this.hostObjects.bridge.CaptureScreen(i).then(r=>f(r))
		return this
	}
	ocr(...a){this.hostObjects.bridge.Scan(...a)
		return this
	}
	async post(url,data,success) {
    	var result = await this.hostObjects.bridge.Post(url,data);
    	success(result)
	}
}

function _html(...a){
	var b = a.shift().split(',')
	var c = b.shift().split('.')
	var d = c.shift()||'div'
	return $(`<${d}>`).addClass(...c).append(...b,...a)
}

</script>
<style>
/* 略 */
</style></head></html>