[Pre-commit] Add pre-commit configuration to perform minimal checks l…

…ocally (#8382)

* [Pre-commit] Add pre-commit hook configuration file

* [Pre-commit] Add header to configuratin file

* [Pre-commit] Add basic configuration instructions

* [Pre-commit] Extend pre-commit pipelines with C++ linting

* [pre-commit] Add example usage comment for pre-commit hooks

* [CI] Add in docker linting script mypy step

* [CI] Use lint docker image for pre-commit checks

* [CI][pre-commit] Minor cleanups on docker runners of pre-commit lints

.pre-commit-config.yaml

@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Pre-commit hook
+# See documentation at: https://pre-commit.com/
+#
+# Pre-commit hook to run the sanity checks from Jenkins locally.
+#
+# Requirements:
+#   - How to configure:
+#        - $ pip install pre-commit
+#        - $ pre-commit install
+#   - How to prevent running it:
+#        - git options: --no-verify or -n
+#        - $ git commit -n -m "YOUR COMMIT MESSAGE"
+#   - How to run it as standalone
+#        - $ pre-commit run
+#
+
+default_language_version:
+    python: python3.8
+fail_fast: True
+repos:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v2.3.0
+      hooks:
+        - id: check-added-large-files
+        - id: check-merge-conflict
+        - id: check-yaml
+        - id: end-of-file-fixer
+        - id: trailing-whitespace
+    - repo: local
+      hooks:
+        -   id: run-black
+            name: Running Black...
+            entry: docker/lint.sh python_format
+            language: system
+            always_run: true
+            pass_filenames: false
+        -   id: run-file-checks
+            name: Checking File Types....
+            entry: docker/lint.sh file_type
+            language: system
+            always_run: true
+            pass_filenames: false
+        -   id: run-headers-check
+            name: Checking ASF License Headers ...
+            entry: docker/lint.sh asf
+            language: system
+            always_run: true
+            pass_filenames: false
+        -   id: run-headers-check
+            name: Linting the C++ code ...
+            entry: docker/lint.sh cpplint
+            language: system
+            always_run: true
+            pass_filenames: false
+        -   id: run-clang-format
+            name: Checking Clang format ...
+            entry: docker/lint.sh clang_format
+            language: system
+            always_run: true
+            pass_filenames: false
+        -   id: run-mypy
+            name: Type Checking with MyPY ...
+            entry: docker/lint.sh mypy
+            language: system
+            always_run: true
+            pass_filenames: false

docker/bash.sh

@@ -20,7 +20,7 @@
 #
 # Start a bash, mount REPO_MOUNT_POINT to be current directory.
 #
-# Usage: docker/bash.sh [-i|--interactive] [--net=host]
+# Usage: docker/bash.sh [-i|--interactive] [--net=host] [-t|--tty]
 #          [--mount MOUNT_DIR] [--repo-mount-point REPO_MOUNT_POINT]
 #          [--dry-run]
 #          <DOCKER_IMAGE_NAME> [--] [COMMAND]
@@ -35,6 +35,7 @@
 
 set -euo pipefail
 
+
 function show_usage() {
     cat <<EOF
 Usage: docker/bash.sh [-i|--interactive] [--net=host]
@@ -50,6 +51,10 @@ Usage: docker/bash.sh [-i|--interactive] [--net=host]
 
     Start the docker session in interactive mode.
 
+-t, --tty
+
+    Start the docker session with a pseudo terminal (tty).
+
 --net=host
 
     Expose servers run into the container to the host, passing the
@@ -108,6 +113,7 @@ REPO_DIR="$(dirname "${SCRIPT_DIR}")"
 
 DRY_RUN=false
 INTERACTIVE=false
+TTY=false
 USE_NET_HOST=false
 DOCKER_IMAGE_NAME=
 COMMAND=bash
@@ -150,6 +156,11 @@ while (( $# )); do
             eval $break_joined_flag
             ;;
 
+        -t*|--tty)
+            TTY=true
+            eval $break_joined_flag
+            ;;
+
         --net=host)
             USE_NET_HOST=true
             shift
@@ -293,7 +304,11 @@ fi
 
 # Set up interactive sessions
 if ${INTERACTIVE}; then
-    DOCKER_FLAGS+=( --interactive --tty )
+    DOCKER_FLAGS+=( --interactive )
+fi
+
+if ${TTY}; then
+    DOCKER_FLAGS+=( --tty )
 fi
 
 # Expose external directories to the docker container
@@ -395,6 +410,7 @@ echo ""
 
 echo Running \'${COMMAND[@]+"${COMMAND[@]}"}\' inside ${DOCKER_IMAGE_NAME}...
 
+
 DOCKER_CMD=(${DOCKER_BINARY} run
             ${DOCKER_FLAGS[@]+"${DOCKER_FLAGS[@]}"}
             ${DOCKER_ENV[@]+"${DOCKER_ENV[@]}"}

docker/dev_common.sh

@@ -55,7 +55,6 @@ function lookup_image_spec() {
     fi
 }
 
-
 function run_docker() {
     image_name="$1"  # Name of the Jenkinsfile var to find
     shift
@@ -66,5 +65,5 @@ function run_docker() {
         exit 2
     fi
 
-    "${GIT_TOPLEVEL}/docker/bash.sh" -i "${image_spec}" "$@"
+    "${GIT_TOPLEVEL}/docker/bash.sh" "${image_spec}" "$@"
 }

docker/lint.sh

@@ -20,7 +20,7 @@
 source "$(dirname $0)/dev_common.sh"
 
 SCRIPT_NAME="$0"
-DEFAULT_STEPS=( file_type asf cpplint clang_format pylint python_format jnilint cppdocs )
+DEFAULT_STEPS=( file_type asf cpplint clang_format pylint python_format jnilint cppdocs mypy )
 
 inplace_fix=0
 
@@ -67,6 +67,9 @@ function run_lint_step() {
         cppdocs)
             cmd=( tests/lint/cppdocs.sh )
             ;;
+        mypy)
+            cmd=( tests/scripts/task_mypy.sh )
+            ;;
         *)
             echo "error: don't know how to run lint step: $1" >&2
             echo "usage: ${SCRIPT_NAME} [-i] <lint_step>" >&2