CLI: Add development mode support

Hide development specific options and don't require dev modules unless
`user.developer` is set to `True`.

8 files changed, 302 insertions, 25 deletions

diff --git a/bin/qmk b/bin/qmk
index 7592eefd93..e4fb057ff2 100755
--- a/bin/qmk
+++ b/bin/qmk
@@ -4,34 +4,58 @@
 import os
 import sys
 from importlib.util import find_spec
+from time import strftime
+from pathlib import Path
 
 # Add the QMK python libs to our path
-script_dir = os.path.dirname(os.path.realpath(__file__))
-qmk_dir = os.path.abspath(os.path.join(script_dir, '..'))
-python_lib_dir = os.path.abspath(os.path.join(qmk_dir, 'lib', 'python'))
-sys.path.append(python_lib_dir)
+script_dir = Path(os.path.realpath(__file__)).parent
+qmk_dir = script_dir.parent
+python_lib_dir = Path(qmk_dir / 'lib' / 'python').resolve()
+sys.path.append(str(python_lib_dir))
+
+# QMK CLI user config file
+config_file = Path(Path.home() / '.config/qmk/qmk.ini')
 
-# Make sure our modules have been setup
-with open(os.path.join(qmk_dir, 'requirements.txt'), 'r') as fd:
-    for line in fd.readlines():
-        line = line.strip().replace('<', '=').replace('>', '=')
 
-        if line[0] == '#':
-            continue
+def _check_modules(requirements):
+    """ Check if the modules in the given requirements.txt are available.
+    """
+    with Path(qmk_dir / requirements).open() as fd:
+        for line in fd.readlines():
+            line = line.strip().replace('<', '=').replace('>', '=')
 
-        if '#' in line:
-            line = line.split('#')[0]
+            if line[0] == '#':
+                continue
 
-        module = line.split('=')[0] if '=' in line else line
+            if '#' in line:
+                line = line.split('#')[0]
+
+            module = dict()
+            module['name'] = module['import'] = line.split('=')[0] if '=' in line else line
 
-        if module in ['pep8-naming']:
             # Not every module is importable by its own name.
-            continue
+            if module['name'] == "pep8-naming":
+                module['import'] = "pep8ext_naming"
 
-        if not find_spec(module):
-            print('Could not find module %s!' % module)
-            print('Please run `pip3 install -r requirements.txt` to install the python dependencies.')
-            exit(255)
+            if not find_spec(module['import']):
+                print('Could not find module %s!' % module['name'])
+                if developer:
+                    print('Please run `pip3 install -r requirements-dev.txt` to install the python development dependencies or turn off developer mode with `qmk config user.developer=None`.')
+                    print()
+                else:
+                    print('Please run `pip3 install -r requirements.txt` to install the python dependencies.')
+                    print()
+                    exit(255)
+
+
+developer = False
+# Make sure our modules have been setup
+_check_modules('requirements.txt')
+
+# For developers additional modules are needed
+if config_file.exists() and 'developer = True' in config_file.read_text():
+    developer = True
+    _check_modules('requirements-dev.txt')
 
 # Setup the CLI
 import milc  # noqa
diff --git a/docs/cli.md b/docs/cli.md
index 760fe1cdb5..625ac4fb70 100644
--- a/docs/cli.md
+++ b/docs/cli.md
@@ -37,3 +37,253 @@ We are looking for people to create and maintain a `qmk` package for more operat
     * Document why in a comment when you do deviate
 * Install using a virtualenv
 * Instruct the user to set the environment variable `QMK_HOME` to have the firmware source checked out somewhere other than `~/qmk_firmware`.
+
+# Local CLI
+
+If you do not want to use the global CLI there is a local CLI bundled with `qmk_firmware`. You can find it in `qmk_firmware/bin/qmk`. You can run the `qmk` command from any directory and it will always operate on that copy of `qmk_firmware`.
+
+**Example**:
+
+```
+$ ~/qmk_firmware/bin/qmk hello
+Ψ Hello, World!
+```
+
+## Local CLI Limitations
+
+There are some limitations to the local CLI compared to the global CLI:
+
+* The local CLI does not support `qmk setup` or `qmk clone`
+* The local CLI always operates on the same `qmk_firmware` tree, even if you have multiple repositories cloned.
+* The local CLI does not run in a virtualenv, so it's possible that dependencies will conflict
+
+# CLI Commands
+
+## `qmk cformat`
+
+*dev mode*
+
+This command formats C code using clang-format. Run it with no arguments to format all core code, or pass filenames on the command line to run it on specific files.
+
+**Usage**:
+
+```
+qmk cformat [file1] [file2] [...] [fileN]
+```
+
+## `qmk compile`
+
+This command allows you to compile firmware from any directory. You can compile JSON exports from <https://config.qmk.fm>, compile keymaps in the repo, or compile the keyboard in the current working directory.
+
+**Usage for Configurator Exports**:
+
+```
+qmk compile <configuratorExport.json>
+```
+
+**Usage for Keymaps**:
+
+```
+qmk compile -kb <keyboard_name> -km <keymap_name>
+```
+
+**Usage in Keyboard Directory**:  
+
+Must be in keyboard directory with a default keymap, or in keymap directory for keyboard, or supply one with `--keymap <keymap_name>`
+```
+qmk compile
+```
+
+**Example**:
+```
+$ qmk config compile.keymap=default
+$ cd ~/qmk_firmware/keyboards/planck/rev6
+$ qmk compile
+Ψ Compiling keymap with make planck/rev6:default
+...
+```
+or with optional keymap argument
+
+```
+$ cd ~/qmk_firmware/keyboards/clueboard/66/rev4 
+$ qmk compile -km 66_iso
+Ψ Compiling keymap with make clueboard/66/rev4:66_iso
+...
+```
+or in keymap directory
+
+```
+$ cd ~/qmk_firmware/keyboards/gh60/satan/keymaps/colemak
+$ qmk compile
+Ψ Compiling keymap with make make gh60/satan:colemak
+...
+```
+
+**Usage in Layout Directory**:  
+
+Must be under `qmk_firmware/layouts/`, and in a keymap folder.
+```
+qmk compile -kb <keyboard_name>
+```
+
+**Example**:
+```
+$ cd ~/qmk_firmware/layouts/community/60_ansi/mechmerlin-ansi
+$ qmk compile -kb dz60
+Ψ Compiling keymap with make dz60:mechmerlin-ansi
+...
+```
+
+## `qmk flash`
+
+This command is similar to `qmk compile`, but can also target a bootloader. The bootloader is optional, and is set to `:flash` by default.
+To specify a different bootloader, use `-bl <bootloader>`. Visit <https://docs.qmk.fm/#/flashing>
+for more details of the available bootloaders.
+
+**Usage for Configurator Exports**:
+
+```
+qmk flash <configuratorExport.json> -bl <bootloader>
+```
+
+**Usage for Keymaps**:
+
+```
+qmk flash -kb <keyboard_name> -km <keymap_name> -bl <bootloader>
+```
+
+**Listing the Bootloaders**
+
+```
+qmk flash -b
+```
+
+## `qmk config`
+
+This command lets you configure the behavior of QMK. For the full `qmk config` documentation see [CLI Configuration](cli_configuration.md).
+
+**Usage**:
+
+```
+qmk config [-ro] [config_token1] [config_token2] [...] [config_tokenN]
+```
+
+## `qmk docs`
+
+This command starts a local HTTP server which you can use for browsing or improving the docs. Default port is 8936.
+
+**Usage**:
+
+```
+qmk docs [-p PORT]
+```
+
+## `qmk doctor`
+
+This command examines your environment and alerts you to potential build or flash problems. It can fix many of them if you want it to.
+
+**Usage**:
+
+```
+qmk doctor [-y] [-n]
+```
+
+**Examples**:
+
+Check your environment for problems and prompt to fix them:
+
+    qmk doctor
+
+Check your environment and automatically fix any problems found:
+
+    qmk doctor -y
+
+Check your environment and report problems only:
+
+    qmk doctor -n
+
+## `qmk json-keymap`
+
+Creates a keymap.c from a QMK Configurator export.
+
+**Usage**:
+
+```
+qmk json-keymap [-o OUTPUT] filename
+```
+
+## `qmk kle2json`
+
+This command allows you to convert from raw KLE data to QMK Configurator JSON. It accepts either an absolute file path, or a file name in the current directory. By default it will not overwrite `info.json` if it is already present. Use the `-f` or `--force` flag to overwrite.
+
+**Usage**:
+
+```
+qmk kle2json [-f] <filename>
+```
+
+**Examples**:
+
+```
+$ qmk kle2json kle.txt 
+☒ File info.json already exists, use -f or --force to overwrite.
+```
+
+```
+$ qmk kle2json -f kle.txt -f
+Ψ Wrote out to info.json
+```
+
+## `qmk list-keyboards`
+
+This command lists all the keyboards currently defined in `qmk_firmware`
+
+**Usage**:
+
+```
+qmk list-keyboards
+```
+
+## `qmk list-keymaps`
+
+This command lists all the keymaps for a specified keyboard (and revision).
+
+**Usage**:
+
+```
+qmk list-keymaps -kb planck/ez
+```
+
+## `qmk new-keymap`
+
+This command creates a new keymap based on a keyboard's existing default keymap.
+
+**Usage**:
+
+```
+qmk new-keymap [-kb KEYBOARD] [-km KEYMAP]
+```
+
+## `qmk pyformat`
+
+*dev mode*
+
+This command formats python code in `qmk_firmware`.
+
+**Usage**:
+
+```
+qmk pyformat
+```
+
+## `qmk pytest`
+
+*dev mode*
+
+This command runs the python test suite. If you make changes to python code you should ensure this runs successfully.
+
+**Usage**:
+
+```
+qmk pytest
+```
diff --git a/docs/cli_development.md b/docs/cli_development.md
index cc8c59d067..e41afc42f4 100644
--- a/docs/cli_development.md
+++ b/docs/cli_development.md
@@ -6,6 +6,8 @@ This document has useful information for developers wishing to write new `qmk` s
 
 The QMK CLI operates using the subcommand pattern made famous by git. The main `qmk` script is simply there to setup the environment and pick the correct entrypoint to run. Each subcommand is a self-contained module with an entrypoint (decorated by `@cli.subcommand()`) that performs some action and returns a shell returncode, or None.
 
+*Tip*: Enable dev mode by `qmk config user.developer=True`
+
 # Subcommands
 
 [MILC](https://github.com/clueboard/milc) is the CLI framework `qmk` uses to handle argument parsing, configuration, logging, and many other features. It lets you focus on writing your tool without wasting your time writing glue code.
diff --git a/lib/python/qmk/cli/hello.py b/lib/python/qmk/cli/hello.py
index bee28c3013..5119188a07 100755
--- a/lib/python/qmk/cli/hello.py
+++ b/lib/python/qmk/cli/hello.py
@@ -6,7 +6,7 @@ from milc import cli
 
 
 @cli.argument('-n', '--name', default='World', help='Name to greet.')
-@cli.subcommand('QMK Hello World.')
+@cli.subcommand('QMK Hello World.', hidden=False if cli.config.user.developer else True)
 def hello(cli):
     """Log a friendly greeting.
     """
diff --git a/lib/python/qmk/cli/pyformat.py b/lib/python/qmk/cli/pyformat.py
index a53ba40c0a..1464443804 100755
--- a/lib/python/qmk/cli/pyformat.py
+++ b/lib/python/qmk/cli/pyformat.py
@@ -5,7 +5,7 @@ from milc import cli
 import subprocess
 
 
-@cli.subcommand("Format python code according to QMK's style.")
+@cli.subcommand("Format python code according to QMK's style.", hidden=False if cli.config.user.developer else True)
 def pyformat(cli):
     """Format python code according to QMK's style.
     """
diff --git a/lib/python/qmk/cli/pytest.py b/lib/python/qmk/cli/pytest.py
index 09611d750f..5417a9cb34 100644
--- a/lib/python/qmk/cli/pytest.py
+++ b/lib/python/qmk/cli/pytest.py
@@ -7,7 +7,7 @@ import subprocess
 from milc import cli
 
 
-@cli.subcommand('QMK Python Unit Tests')
+@cli.subcommand('QMK Python Unit Tests', hidden=False if cli.config.user.developer else True)
 def pytest(cli):
     """Run several linting/testing commands.
     """
diff --git a/requirements-dev.txt b/requirements-dev.txt
new file mode 100644
index 0000000000..deab419cb2
--- /dev/null
+++ b/requirements-dev.txt
@@ -0,0 +1,4 @@
+# Python development requirements
+nose2
+flake8
+pep8-naming
diff --git a/requirements.txt b/requirements.txt
index 074b11a8ce..aa6ee1ba32 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,6 +4,3 @@ appdirs
 argcomplete
 colorama
 hjson
-nose2
-flake8
-pep8-naming