ゲーム翻訳ツール

　

　
サンプルコード

GAS

function doPost(e) {
    var a = [e.postData.getDataAsString(),'en','ja']
    return ContentService.createTextOutput(LanguageApp.translate(...a))
}

MainWindow.xaml.cs

namespace Konjac{
    public partial class MainWindow : Window{
        public MainWindow(){
            InitializeComponent();
            webView.NavigationCompleted += (s, e) => {
                webView.CoreWebView2.AddHostObjectToScript("bridge", new Bridge(webView));
            };
            webView.Source = new Uri(GetPath() + "\\Konjac.html");
        }
        private static string GetPath(){
            var p = System.Diagnostics.Process.GetCurrentProcess();
            return System.IO.Path.GetDirectoryName(p.MainModule.FileName);
        }
    }
}

Bridge.cs

public class Native{
    // DPI (PerMonitorV2)
    //https://msdn.microsoft.com/en-us/library/windows/desktop/dd145062(v=vs.85).aspx
    //https://msdn.microsoft.com/en-us/library/windows/desktop/dn280510(v=vs.85).aspx
    public const int WM_DPICHANGED = 0x02E0;
    [DllImport("User32.dll")]
    public static extern IntPtr MonitorFromPoint([In] Point pt, [In] uint dwFlags);
    [DllImport("Shcore.dll")]
    public static extern IntPtr GetDpiForMonitor([In] IntPtr hmonitor, [In] DpiType dpiType, [Out] out uint dpiX, [Out] out uint dpiY);
    public enum DpiType{
        Effective = 0,
        Angular = 1,
        Raw = 2,
    }
}
public class ScreenData{
    public uint dpi { get; set; }
    public Screen screen { get; set; }
    public ScreenData(Screen s){
        var p = new Point(s.Bounds.Left + 1, s.Bounds.Top + 1);
        var m = Native.MonitorFromPoint(p, 2/*MONITOR_DEFAULTTONEAREST*/);
        uint x, y;
        Native.GetDpiForMonitor(m, Native.DpiType.Effective, out x, out y);
        screen = s;
        dpi = x;
    }
}
public class BridgeEvent{
    public string type { get; set; }
    public string value { get; set; }
    public BridgeEvent(string t, string v){
        type = t;
        value = v;
    }
    public string ToJson(){
        return JsonSerializer.Serialize(this);
    }
}

[ComVisible(true)]
public class Bridge{
    Bitmap capture;
    WebView2 view;
    public Bridge(WebView2 w){
        view = w;
    }
    public async void Scan(int x, int y, int w, int h){
        Bitmap textBmp = new Bitmap(w, h);
        Graphics g = Graphics.FromImage(textBmp);
        g.DrawImage(capture, new Point(-x, -y));
        g.Dispose();
        string file = "Konjac.png";
        string path = $"{GetPath()}\\{file}";
        textBmp.Save(path);
        string s = await OCR(file);
        Console.WriteLine(s);
        var e = new BridgeEvent("SCAN_DONE", s);

        //jsからの非同期呼び出しでnullが返る
        //https://github.com/MicrosoftEdge/WebView2Feedback/issues/822
        view.CoreWebView2.PostWebMessageAsString(e.ToJson());
    }
    public async Task<string> OCR(string file){
        //https://water2litter.net/rye/post/c_ocr_hello/

        Stream input_stream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read);
        var source = new WriteableBitmap(System.Windows.Media.Imaging.BitmapFrame.Create(input_stream));
        input_stream.Close();

        PngBitmapEncoder encoder = new PngBitmapEncoder();
        encoder.Frames.Add(System.Windows.Media.Imaging.BitmapFrame.Create(source));
        MemoryStream temp_stream = new MemoryStream();
        encoder.Save(temp_stream);

        var converted_stream = WindowsRuntimeStreamExtensions.AsRandomAccessStream(temp_stream);

        var decorder = await Windows.Graphics.Imaging.BitmapDecoder.CreateAsync(converted_stream);
        SoftwareBitmap bitmap = await decorder.GetSoftwareBitmapAsync();
        converted_stream.Dispose();
        temp_stream.Close();

        OcrEngine engine = OcrEngine.TryCreateFromLanguage(new Windows.Globalization.Language("en"));
        var result = await engine.RecognizeAsync(bitmap);
        return result.Text;
    }
    public string Post(string url, string data){
        //https://docs.microsoft.com/ja-jp/dotnet/framework/network-programming/how-to-send-data-using-the-webrequest-class

        Console.WriteLine(url);
        byte[] byteArray = Encoding.UTF8.GetBytes(data);
        var request = (HttpWebRequest)WebRequest.Create(url);
        request.Method = "POST";
        request.ContentType = "application/x-www-form-urlencoded";
        request.ContentLength = byteArray.Length;
        Stream dataStream = request.GetRequestStream();
        dataStream.Write(byteArray, 0, byteArray.Length);
        dataStream.Close();

        var response = (HttpWebResponse)request.GetResponse();
        string responseContent = string.Empty;
        using (dataStream = response.GetResponseStream())
        {
            StreamReader reader = new StreamReader(dataStream);
            string responseFromServer = reader.ReadToEnd();
            Console.WriteLine(responseFromServer);
            response.Close();
            return responseFromServer;
        }
    }
    public string CaptureScreen(int i){
        Screen s = Screen.AllScreens[i];
        capture = new Bitmap(s.Bounds.Width, s.Bounds.Height);
        Graphics g = Graphics.FromImage(capture);
        g.CopyFromScreen(new Point(s.Bounds.X, s.Bounds.Y), new Point(0, 0), capture.Size);
        g.Dispose();
        string file = $"Screen{i}.png";
        string path = $"{GetPath()}\\{file}";
        Console.WriteLine(path);
        capture.Save(path);
        return file;
    }
    private static string GetPath(){
        var p = System.Diagnostics.Process.GetCurrentProcess();
        return Path.GetDirectoryName(p.MainModule.FileName);
    }
    public string getAllScreens(){
        List<ScreenData> screens = new List<ScreenData>();
        foreach (Screen s in Screen.AllScreens){
            screens.Add(new ScreenData(s));
        }
        return JsonSerializer.Serialize(screens);
    }
}

Konjac.html

<!DOCTYPE html><html lang="ja"><head>
<meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/1.12.4/jquery.min.js"></script>
<script>

DEPLOY_ID = '***************************' //GAS

$(e=>{$('body').append(new Konjac().view)})

class Konjac{
    constructor(){
        Konjac.native = new Konjac.Bridge().ready(v=>{
            v.getScreens(r=>{
                this.item = r.map((v,i)=>new Konjac.Screen(v,i))
                this.init()
            })
        })
        this.view = _html('.container')
    }
    init(){
        this.currentItem = this.item[0]
        var screens = this.item.sort((a,b)=>{return a.x-b.x})
        screens.forEach(v=>v.menuItem.click(e=>this.select(v)))

        this.view.append(
            _html('.content',screens.map(v=>v.view)),
            this.overlay = _html('.overlay').hide().click(e=>this.overlay.hide()),
            _html('.header',
                _html('ul',screens.map(v=>v.menuItem)),
                _html('.button,翻訳する').click(e=>{
                    this.overlay.empty().show()
                    this.ocr()
                })
            ),
        )
        Konjac.native.hook.SCAN_DONE = e=>{
            this.overlay.append(_html('.en',e.value))
            this.translate(e.value,ja=>{
                this.overlay.prepend(_html('.ja',ja))
            })
        }
        var g = new Konjac.Gamepad({
            GAMEPAD_A:e=>this.rescan(),
            GAMEPAD_B:e=>{this.overlay.hide()}
        })
        this.select(this.currentItem)
    }
    rescan(){
        this.currentItem.capture(r=>{
            this.overlay.empty().show()
            this.ocr()
        })
    }
    select(item){
        this.overlay.hide()
        this.currentItem = item
        this.item.forEach(v=>{
            v.menuItem.removeClass('selected')
            v.view.hide()
        })
        item.menuItem.addClass('selected')
        item.view.show()
        item.capture()
    }
    ocr(){
        var dpr = this.currentItem.dpr
        var w = $(window)
        var rect = [    
            Math.round(w.scrollLeft()*dpr),
            Math.round(w.scrollTop()*dpr),
            Math.round(w.width()*dpr),
            Math.round((w.height()-$('.header').height())*dpr),
        ]
        Konjac.native.ocr(...rect)
        return this
    }
    translate(data,success){
        var url = `https://script.google.com/macros/s/${DEPLOY_ID}/exec`;
        Konjac.native.post(url,data,v=>success(v))
        return this
    }
}

Konjac.Gamepad = class KonjacGamepad{
    constructor(hook){
        var s = 'A,B,X,Y,LB,RB,LT,RT,BACK,START,LS,RS,UP,DOWN,LEFT,RIGHT,HOME'
        this.types  = s.split(',').map(s=>'GAMEPAD_'+s)
        this.hook   = hook
        this.stack  = []
        this.length = 0
        window.addEventListener("gamepadconnected",e=>{
            console.log(e)
            this.stack[e.gamepad.index] = [e.gamepad]
            this.length++
        });
        window.addEventListener("gamepaddisconnected",e=>{
            console.log(e)
            this.stack[e.gamepad.index] = []
            this.length--
        });
        setInterval(e=>this.loop(),1000/30)
    }
    loop(){
        if(this.length==0) return
        var items = [...navigator.getGamepads()].filter(Boolean)
        if(items.length==0) return
        var a = this.stack[items[0].index]
        a.unshift(items[0])
        a[0].buttons.forEach((b,i)=>{
            if(b.pressed==true) return
            if(a[1].buttons[i].pressed==false) return
            this.dispatch(this.types[i])
        })
        this.stack[items[0].index] = a.slice(0,1)
    }
    dispatch(type){
        console.log(type)
        if(this.hook[type]) this.hook[type]()
    }
}

Konjac.Screen = class KonjacScreen{
    constructor(data,i){
        var b = data.screen.Bounds
        var dpr = data.dpi/96
        this.x  = b.Left
        this.y  = b.Top
        this.w  = b.Width
        this.h  = b.Height
        this.width  = Math.round(b.Width/dpr)
        this.height = Math.round(b.Height/dpr)
        this.dpr    = dpr//devicePixelRatio
        this.name   = data.screen.DeviceName.replace(/[^\w\s]/g,'')
        this.index  = i
        Object.assign(this,data)
        var s = b.Width+' x '+b.Height
        this.sizeText = Konjac.Screen.type[s]||s
        this.menuItem = _html('li',this.name,' ',_html('span',this.sizeText))
        this.view = _html('img').css({width:this.width})
    }
    capture(f){
        Konjac.native.capture(this.index,r=>{
            this.view.attr('src',r+'?'+Math.random())
            if(f) f(r)
        })
        return this
    }
}
Konjac.Screen.type = {
    '1920 x 1080':'FHD',
    '1920 x 1200':'WUXGA',
    '2560 x 1440':'WQHD',
    '3840 x 2160':'4K',
}

Konjac.Bridge = class KonjacBridge{
    constructor(){
        this.hook = {}
        this.on(()=>{}).ready(()=>{})
        this.hostObjects = {bridge:{}}
        if(chrome.webview){
            this.hostObjects = chrome.webview.hostObjects
            chrome.webview.addEventListener('message',e=>{
                var data = JSON.parse(e.data)
                var hook = this.hook[data.type]||this.hook.on
                hook(data)
            })
        }
        setTimeout(e=>this.hook.ready(this),100)
    }
    on(f){this.hook.on = f
        return this
    }
    ready(f){this.hook.ready = f
        return this
    }
    getScreens(f){
        this.hostObjects.bridge.getAllScreens().then(r=>{
            f(JSON.parse(r))
        }).catch(console.log)
        return this
    }
    capture(i,f){
        this.hostObjects.bridge.CaptureScreen(i).then(r=>f(r))
        return this
    }
    ocr(...a){this.hostObjects.bridge.Scan(...a)
        return this
    }
    async post(url,data,success) {
        var result = await this.hostObjects.bridge.Post(url,data);
        success(result)
    }
}

function _html(...a){
    var b = a.shift().split(',')
    var c = b.shift().split('.')
    var d = c.shift()||'div'
    return $(`<${d}>`).addClass(...c).append(...b,...a)
}

</script>
<style>
/* 略 */
</style></head></html>