Add mobile keyboard API (#21)

* fix page for mobile - minor changes.

* fix textarea overlay to hide caret and avodi zooming on mobiles.

* fix typo.

* show keyboard btn if is touch device.

* lint fix.

* add to API.

* mobile keybaord fix andorid blur.

* add mobile keybaord toggle.

* fix overlay.

* mobile keybaord, skip if not a touch device.

Author: m1k1o

README.md

@@ -26,9 +26,9 @@ echo \"@demodesk:registry\" \"https://npm.pkg.github.com\" >> .yarnrc
 You can set keyboard provider at build time, either `novnc` or the default `guacamole`.
 
 ```bash
-# by default uses guacamole keybaord
+# by default uses guacamole keyboard
 npm run build
-# uses novnc keybaord
+# uses novnc keyboard
 KEYBOARD=novnc npm run build
 ```
 

src/component/main.vue

@@ -20,6 +20,7 @@
         :cursorDraw="inactiveCursorDrawFunction"
       />
       <neko-overlay
+        ref="overlay"
         v-show="!private_mode_enabled && state.connection.status != 'disconnected'"
         :style="{ pointerEvents: state.control.locked ? 'none' : 'auto' }"
         :wsControl="control"
@@ -35,6 +36,7 @@
         :inactiveCursors="state.settings.inactive_cursors && session.profile.sends_inactive_cursor"
         @updateKeyboardModifiers="updateKeyboardModifiers($event)"
         @uploadDrop="uploadDrop($event)"
+        @mobileKeyboardOpen="state.mobile_keyboard_open = $event"
       />
     </div>
   </div>
@@ -102,6 +104,7 @@
     @Ref('component') readonly _component!: HTMLElement
     @Ref('container') readonly _container!: HTMLElement
     @Ref('video') readonly _video!: HTMLVideoElement
+    @Ref('overlay') readonly _overlay!: Overlay
 
     // fallback image for webrtc reconnections:
     // chrome shows black screen when closing webrtc connection, that's why
@@ -196,6 +199,7 @@
         merciful_reconnect: false,
       },
       cursors: [],
+      mobile_keyboard_open: false,
     } as NekoState
 
     /////////////////////////////
@@ -405,6 +409,22 @@
       Vue.set(this.state.control, 'keyboard', { layout, variant })
     }
 
+    public mobileKeyboardShow() {
+      this._overlay.mobileKeyboardShow()
+    }
+
+    public mobileKeyboardHide() {
+      this._overlay.mobileKeyboardHide()
+    }
+
+    public mobileKeyboardToggle() {
+      if (this.state.mobile_keyboard_open) {
+        this.mobileKeyboardHide()
+      } else {
+        this.mobileKeyboardShow()
+      }
+    }
+
     public setCursorDrawFunction(fn?: CursorDrawFunction) {
       Vue.set(this, 'cursorDrawFunction', fn)
     }

src/component/overlay.vue

@@ -33,8 +33,9 @@
     bottom: 0;
     width: 100%;
     height: 100%;
-    font-size: 1px; /* chrome would not paste text if 0px */
+    font-size: 16px; /* at least 16px to avoid zooming on mobile */
     resize: none; /* hide textarea resize corner */
+    caret-color: transparent; /* hide caret */
     outline: 0;
     border: 0;
     color: transparent;
@@ -116,6 +117,10 @@
       return 'url(' + uri + ') ' + x + ' ' + y + ', default'
     }
 
+    get isTouchDevice(): boolean {
+      return 'ontouchstart' in window || navigator.maxTouchPoints > 0
+    }
+
     mounted() {
       // register mouseup globally as user can release mouse button outside of overlay
       window.addEventListener('mouseup', this.onMouseUp, true)
@@ -373,7 +378,11 @@
     }
 
     onMouseEnter(e: MouseEvent) {
-      this._textarea.focus()
+      // focus opens the keyboard on mobile (only for android)
+      if (!this.isTouchDevice) {
+        this._textarea.focus()
+      }
+
       this.focused = true
 
       if (this.isControling) {
@@ -629,5 +638,48 @@
         this.wsControl.release()
       }
     }
+
+    //
+    // mobile keyboard
+    //
+
+    public kbdShow = false
+    public kbdOpen = false
+
+    public mobileKeyboardShow() {
+      // skip if not a touch device
+      if (!this.isTouchDevice) return
+
+      this.kbdShow = true
+      this.kbdOpen = false
+
+      this._textarea.focus()
+      window.visualViewport.addEventListener('resize', this.onVisualViewportResize)
+      this.$emit('mobileKeyboardOpen', true)
+    }
+
+    public mobileKeyboardHide() {
+      // skip if not a touch device
+      if (!this.isTouchDevice) return
+
+      this.kbdShow = false
+      this.kbdOpen = false
+
+      this.$emit('mobileKeyboardOpen', false)
+      window.visualViewport.removeEventListener('resize', this.onVisualViewportResize)
+      this._textarea.blur()
+    }
+
+    // visual viewport resize event is fired when keyboard is opened or closed
+    // android does not blur textarea when keyboard is closed, so we need to do it manually
+    onVisualViewportResize() {
+      if (!this.kbdShow) return
+
+      if (!this.kbdOpen) {
+        this.kbdOpen = true
+      } else {
+        this.mobileKeyboardHide()
+      }
+    }
   }
 </script>

src/component/types/state.ts

@@ -11,6 +11,7 @@ export default interface State {
   sessions: Record<string, Session>
   settings: Settings
   cursors: Cursors
+  mobile_keyboard_open: boolean
 }
 
 /////////////////////////////

src/page/components/events.vue

@@ -400,6 +400,18 @@
         <td>{{ neko.state.cursors }}</td>
       </tr>
 
+      <tr>
+        <th>mobile_keyboard_open</th>
+        <td>
+          <div class="space-between">
+            <span>{{ neko.state.mobile_keyboard_open }}</span>
+            <button @click="neko.mobileKeyboardToggle">
+              <i class="fas fa-toggle-on"></i>
+            </button>
+          </div>
+        </td>
+      </tr>
+
       <tr>
         <th>control actions</th>
         <td>

src/page/main.vue

@@ -43,6 +43,13 @@
         </div>
       </div>
       <div class="room-container" style="text-align: center">
+        <button
+          v-if="loaded && isTouchDevice"
+          @click="neko.mobileKeyboardToggle"
+          style="position: absolute; left: 5px; transform: translateY(-100%)"
+        >
+          <i class="fa fa-keyboard" />
+        </button>
         <span v-if="loaded && neko.state.session_id" style="padding-top: 10px">
           You are logged in as
           <strong style="font-weight: bold">
@@ -193,6 +200,9 @@
       flex-shrink: 0;
       flex-direction: column;
       display: flex;
+      /* for mobile */
+      overflow-y: hidden;
+      overflow-x: auto;
 
       .room-menu {
         max-width: 100%;
@@ -279,10 +289,14 @@
     }
   }
 
+  /* for mobile */
   @media only screen and (max-width: 600px) {
+    $offset: 38px;
+
     #neko.expanded {
+      /* show only enough of the menu to see the toggle button */
       .neko-main {
-        transform: translateX(calc(-100% + 65px));
+        transform: translateX(calc(-100% + $offset));
         video {
           display: none;
         }
@@ -292,14 +306,14 @@
         top: 0;
         right: 0;
         bottom: 0;
-        left: 65px;
-        width: calc(100% - 65px);
+        left: $offset;
+        width: calc(100% - $offset);
+      }
+      /* display menu toggle button far right */
+      .header .menu,
+      .header .menu li {
+        margin-right: 2px;
       }
-    }
-  }
-  @media only screen and (max-width: 768px) {
-    #neko .neko-main .room-container {
-      display: none;
     }
   }
 </style>
@@ -362,7 +376,7 @@
   })
   export default class extends Vue {
     @Ref('neko') readonly neko!: NekoCanvas
-    expanded: boolean = true
+    expanded: boolean = !window.matchMedia('(max-width: 600px)').matches // default to expanded on bigger screens
     loaded: boolean = false
     tab: string = ''
 
@@ -371,6 +385,10 @@
     uploadActive = false
     uploadProgress = 0
 
+    get isTouchDevice(): boolean {
+      return 'ontouchstart' in window || navigator.maxTouchPoints > 0
+    }
+
     dialogOverlayActive = false
     dialogRequestActive = false
     async dialogUploadFiles(files: File[]) {