[CI] Update static checks, copyright headers.

3 files changed, 123 insertions, 89 deletions

diff --git a/misc/scripts/clang_format.sh b/misc/scripts/clang_format.sh
index 63c66d4..74a8e1a 100755
--- a/misc/scripts/clang_format.sh
+++ b/misc/scripts/clang_format.sh
@@ -4,55 +4,50 @@
 # This is the primary script responsible for fixing style violations.
 
 set -uo pipefail
-IFS=$'\n\t'
 
-CLANG_FORMAT_FILE_EXTS=(".c" ".h" ".cpp" ".hpp" ".cc" ".hh" ".cxx" ".m" ".mm" ".inc" ".java" ".glsl")
+if [ $# -eq 0 ]; then
+    # Loop through all code files tracked by Git.
+    files=$(git ls-files -- '*.c' '*.h' '*.cpp' '*.hpp' '*.cc' '*.hh' '*.cxx' '*.m' '*.mm' '*.inc' '*.java' '*.glsl' \
+                ':!:.git/*' ':!:thirdparty/*' ':!:*/thirdparty/*' ':!:platform/android/java/lib/src/com/google/*' \
+                ':!:*-so_wrap.*' ':!:tests/python_build/*')
+else
+    # $1 should be a file listing file paths to process. Used in CI.
+    files=$(cat "$1" | grep -v "thirdparty/" | grep -E "\.(c|h|cpp|hpp|cc|hh|cxx|m|mm|inc|java|glsl)$" | grep -v "platform/android/java/lib/src/com/google/" | grep -v "\-so_wrap\." | grep -v "tests/python_build/")
+fi
+
+if [ ! -z "$files" ]; then
+    clang-format --Wno-error=unknown -i $files
+fi
 
-# Loops through all text files tracked by Git.
-git grep -zIl '' |
-while IFS= read -rd '' f; do
-    # Exclude some files.
-    if [[ "$f" == "thirdparty"* ]]; then
+# Fix copyright headers, but not all files get them.
+for f in $files; do
+    if [[ "$f" == *"inc" ]]; then
         continue
-    elif [[ "$f" == "platform/android/java/lib/src/com/google"* ]]; then
+    elif [[ "$f" == *"glsl" ]]; then
         continue
-    elif [[ "$f" == *"-so_wrap."* ]]; then
+    elif [[ "$f" == "platform/android/java/lib/src/org/godotengine/godot/gl/GLSurfaceView"* ]]; then
+        continue
+    elif [[ "$f" == "platform/android/java/lib/src/org/godotengine/godot/gl/EGLLogWrapper"* ]]; then
+        continue
+    elif [[ "$f" == "platform/android/java/lib/src/org/godotengine/godot/utils/ProcessPhoenix"* ]]; then
         continue
     fi
 
-    for extension in ${CLANG_FORMAT_FILE_EXTS[@]}; do
-        if [[ "$f" == *"$extension" ]]; then
-            # Run clang-format.
-            clang-format -i "$f"
-            # Fix copyright headers, but not all files get them.
-            if [[ "$f" == *"inc" ]]; then
-                continue 2
-            elif [[ "$f" == *"glsl" ]]; then
-                continue 2
-            elif [[ "$f" == *"theme_data.h" ]]; then
-                continue 2
-            elif [[ "$f" == "platform/android/java/lib/src/org/godotengine/godot/input/InputManager"* ]]; then
-                continue 2
-            fi
-            python misc/scripts/copyright_headers.py "$f"
-            continue 2
-        fi
-    done
+    python misc/scripts/copyright_headers.py "$f"
 done
 
-git diff > patch.patch
+diff=$(git diff --color)
 
-# If no patch has been generated all is OK, clean up, and exit.
-if [ ! -s patch.patch ] ; then
-    printf "Files in this commit comply with the clang-format style rules.\n"
-    rm -f patch.patch
+# If no diff has been generated all is OK, clean up, and exit.
+if [ -z "$diff" ] ; then
+    printf "\e[1;32m*** Files in this commit comply with the clang-format style rules.\e[0m\n"
     exit 0
 fi
 
-# A patch has been created, notify the user, clean up, and exit.
-printf "\n*** The following differences were found between the code "
-printf "and the formatting rules:\n\n"
-cat patch.patch
-printf "\n*** Aborting, please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\n"
-rm -f patch.patch
+# A diff has been created, notify the user, clean up, and exit.
+printf "\n\e[1;33m*** The following changes must be made to comply with the formatting rules:\e[0m\n\n"
+# Perl commands replace trailing spaces with `·` and tabs with `<TAB>`.
+printf "$diff\n" | perl -pe 's/(.*[^ ])( +)(\e\[m)$/my $spaces="·" x length($2); sprintf("$1$spaces$3")/ge' | perl -pe 's/(.*[^\t])(\t+)(\e\[m)$/my $tabs="<TAB>" x length($2); sprintf("$1$tabs$3")/ge'
+
+printf "\n\e[1;91m*** Please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\e[0m\n"
 exit 1
diff --git a/misc/scripts/copyright_headers.py b/misc/scripts/copyright_headers.py
index 2f3e4a1..a5e2f0c 100755
--- a/misc/scripts/copyright_headers.py
+++ b/misc/scripts/copyright_headers.py
@@ -4,35 +4,35 @@
 import sys
 
 header = """\
-/*************************************************************************/
-/*  $filename                                                            */
-/*************************************************************************/
-/*                       This file is part of:                           */
-/*                           GODOT ENGINE                                */
-/*                      https://godotengine.org                          */
-/*************************************************************************/
-/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur.                 */
-/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md).   */
-/*                                                                       */
-/* Permission is hereby granted, free of charge, to any person obtaining */
-/* a copy of this software and associated documentation files (the       */
-/* "Software"), to deal in the Software without restriction, including   */
-/* without limitation the rights to use, copy, modify, merge, publish,   */
-/* distribute, sublicense, and/or sell copies of the Software, and to    */
-/* permit persons to whom the Software is furnished to do so, subject to */
-/* the following conditions:                                             */
-/*                                                                       */
-/* The above copyright notice and this permission notice shall be        */
-/* included in all copies or substantial portions of the Software.       */
-/*                                                                       */
-/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
-/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
-/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
-/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
-/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
-/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
-/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
-/*************************************************************************/
+/**************************************************************************/
+/*  $filename                                                             */
+/**************************************************************************/
+/*                         This file is part of:                          */
+/*                             GODOT ENGINE                               */
+/*                        https://godotengine.org                         */
+/**************************************************************************/
+/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */
+/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur.                  */
+/*                                                                        */
+/* Permission is hereby granted, free of charge, to any person obtaining  */
+/* a copy of this software and associated documentation files (the        */
+/* "Software"), to deal in the Software without restriction, including    */
+/* without limitation the rights to use, copy, modify, merge, publish,    */
+/* distribute, sublicense, and/or sell copies of the Software, and to     */
+/* permit persons to whom the Software is furnished to do so, subject to  */
+/* the following conditions:                                              */
+/*                                                                        */
+/* The above copyright notice and this permission notice shall be         */
+/* included in all copies or substantial portions of the Software.        */
+/*                                                                        */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,        */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF     */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY   */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,   */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE      */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                 */
+/**************************************************************************/
 """
 
 fname = sys.argv[1]
diff --git a/misc/scripts/file_format.sh b/misc/scripts/file_format.sh
index 3bb4d8d..94a3aff 100755
--- a/misc/scripts/file_format.sh
+++ b/misc/scripts/file_format.sh
@@ -4,27 +4,52 @@
 # This is supplementary to clang_format.sh and black_format.sh, but should be
 # run before them.
 
-# We need dos2unix and recode.
-if [ ! -x "$(command -v dos2unix)" -o ! -x "$(command -v recode)" ]; then
-    printf "Install 'dos2unix' and 'recode' to use this script.\n"
+# We need dos2unix and isutf8.
+if [ ! -x "$(command -v dos2unix)" -o ! -x "$(command -v isutf8)" ]; then
+    printf "Install 'dos2unix' and 'isutf8' (moreutils package) to use this script.\n"
+    exit 1
 fi
 
 set -uo pipefail
-IFS=$'\n\t'
 
-# Loops through all text files tracked by Git.
-git grep -zIl '' |
-while IFS= read -rd '' f; do
+if [ $# -eq 0 ]; then
+    # Loop through all code files tracked by Git.
+    mapfile -d '' files < <(git grep -zIl '')
+else
+    # $1 should be a file listing file paths to process. Used in CI.
+    mapfile -d ' ' < <(cat "$1")
+fi
+
+for f in "${files[@]}"; do
     # Exclude some types of files.
-    if [[ "$f" == "webrtc"* ]]; then
+    if [[ "$f" == *"csproj" ]]; then
+        continue
+    elif [[ "$f" == *"sln" ]]; then
+        continue
+    elif [[ "$f" == *".bat" ]]; then
+        continue
+    elif [[ "$f" == *".out" ]]; then
+        # GDScript integration testing files.
+        continue
+    elif [[ "$f" == *"patch" ]]; then
+        continue
+    elif [[ "$f" == *"pot" ]]; then
+        continue
+    elif [[ "$f" == *"po" ]]; then
         continue
-    elif [[ "$f" == "godot-cpp"* ]]; then
+    elif [[ "$f" == "thirdparty/"* ]]; then
         continue
-    elif [[ "$f" == "misc/patches"* ]]; then
+    elif [[ "$f" == *"/thirdparty/"* ]]; then
+        continue
+    elif [[ "$f" == "platform/android/java/lib/src/com/google"* ]]; then
+        continue
+    elif [[ "$f" == *"-so_wrap."* ]]; then
+        continue
+    elif [[ "$f" == *".test.txt" ]]; then
         continue
     fi
     # Ensure that files are UTF-8 formatted.
-    recode UTF-8 "$f" 2> /dev/null
+    isutf8 "$f" >> utf8-validation.txt 2>&1
     # Ensure that files have LF line endings and do not contain a BOM.
     dos2unix "$f" 2> /dev/null
     # Remove trailing space characters and ensures that files end
@@ -32,19 +57,33 @@ while IFS= read -rd '' f; do
     perl -i -ple 's/\s*$//g' "$f"
 done
 
-git diff > patch.patch
+diff=$(git diff --color)
 
-# If no patch has been generated all is OK, clean up, and exit.
-if [ ! -s patch.patch ] ; then
-    printf "Files in this commit comply with the formatting rules.\n"
-    rm -f patch.patch
+if [ ! -s utf8-validation.txt ] && [ -z "$diff" ] ; then
+    # If no UTF-8 violations were collected (the file is empty) and
+    # no diff has been generated all is OK, clean up, and exit.
+    printf "\e[1;32m*** Files in this commit comply with the file formatting rules.\e[0m\n"
+    rm -f utf8-validation.txt
     exit 0
 fi
 
-# A patch has been created, notify the user, clean up, and exit.
-printf "\n*** The following differences were found between the code "
-printf "and the formatting rules:\n\n"
-cat patch.patch
-printf "\n*** Aborting, please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\n"
-rm -f patch.patch
+if [ -s utf8-validation.txt ]
+then
+    # If the file has content and is not empty, violations
+    # detected, notify the user, clean up, and exit.
+    printf "\n\e[1;33m*** The following files contain invalid UTF-8 character sequences:\e[0m\n\n"
+    cat utf8-validation.txt
+fi
+
+rm -f utf8-validation.txt
+
+if [ ! -z "$diff" ]
+then
+    # A diff has been created, notify the user, clean up, and exit.
+    printf "\n\e[1;33m*** The following changes must be made to comply with the formatting rules:\e[0m\n\n"
+    # Perl commands replace trailing spaces with `·` and tabs with `<TAB>`.
+    printf "$diff\n" | perl -pe 's/(.*[^ ])( +)(\e\[m)$/my $spaces="·" x length($2); sprintf("$1$spaces$3")/ge' | perl -pe 's/(.*[^\t])(\t+)(\e\[m)$/my $tabs="<TAB>" x length($2); sprintf("$1$tabs$3")/ge'
+fi
+
+printf "\n\e[1;91m*** Please fix your commit(s) with 'git commit --amend' or 'git rebase -i <hash>'\e[0m\n"
 exit 1