From 1230c10bd4ea489685c5319c1699a697cde66bed Mon Sep 17 00:00:00 2001 From: Bruno Rocha Date: Fri, 23 Sep 2016 13:05:23 -0300 Subject: [PATCH] improved uuid checks on commit hook and make targets --- Makefile | 29 ++++----- scripts/check_duplicate_uuids.sh | 27 -------- scripts/fix_uuids.sh | 107 +++++++++++++++++++++++++++++++ scripts/replace_dup_uuids.sh | 20 ------ scripts/replace_empty_uuids.sh | 25 -------- 5 files changed, 120 insertions(+), 88 deletions(-) delete mode 100755 scripts/check_duplicate_uuids.sh create mode 100755 scripts/fix_uuids.sh delete mode 100755 scripts/replace_dup_uuids.sh delete mode 100755 scripts/replace_empty_uuids.sh diff --git a/Makefile b/Makefile index 02d396cab7e..211cad01e04 100644 --- a/Makefile +++ b/Makefile @@ -47,11 +47,11 @@ help: @echo " logs-join to join xdist log files into one" @echo " logs-clean to delete all xdist log files in the root" @echo " pyc-clean to delete all temporary artifacts" - @echo " uuid-check to check for duplicated @id: in testimony docstring tags" - @echo " uuid-replace-empty to replace empty @id: with new generated uuid" - @echo " uuid-replace-duplicate to replace duplicated @id: with new generated uuid" + @echo " uuid-check to check for duplicated or empty @id: in testimony docstring tags" + @echo " uuid-fix to fix all duplicated or empty @id: in testimony docstring tags" @echo " can-i-push? to check if local changes are suitable to push" @echo " install-commit-hook to install pre-commit hook to check if changes are suitable to push" + @echo " gitflake8 to check flake8 styling only for modified files" docs: @cd docs; $(MAKE) html @@ -132,26 +132,24 @@ logs-join: logs-clean: -rm -f robottelo_gw*.log -uuid-check: ## list duplicated uuids +uuid-check: ## list duplicated or empty uuids $(info "Checking for empty or duplicated @id: in docstrings...") - scripts/check_duplicate_uuids.sh + @scripts/fix_uuids.sh --check -uuid-replace-duplicate: ## list duplicated uuids - scripts/replace_dup_uuids.sh +uuid-fix: + @scripts/fix_uuids.sh -uuid-replace-empty: ## list duplicated uuids - scripts/replace_empty_uuids.sh - -flake8: +gitflake8: $(info "Checking style and syntax errors with flake8 linter...") - @flake8 . --show-source + @flake8 $(shell git diff --name-only) --show-source -can-i-push?: flake8 uuid-check test-docstrings test-robottelo +can-i-push?: gitflake8 uuid-check test-docstrings test-robottelo $(info "!!! Congratulations your changes are good to fly, make a great PR! ${USER}++ !!!") install-commit-hook: $(info "Installing git pre-commit hook...") - echo "make can-i-push?" >> .git/hooks/pre-commit + @grep -q '^make uuid-fix' .git/hooks/pre-commit || echo "make uuid-fix" >> .git/hooks/pre-commit + @grep -q '^make can-i-push?' .git/hooks/pre-commit || echo "make can-i-push?" >> .git/hooks/pre-commit # Special Targets ------------------------------------------------------------- @@ -161,5 +159,4 @@ install-commit-hook: test-foreman-tier2 test-foreman-tier3 test-foreman-tier4 \ test-foreman-ui test-foreman-ui-xvfb test-foreman-endtoend \ graph-entities lint logs-join logs-clean pyc-clean \ - uuid-check uuid-replace-duplicate uuid-replace-empty \ - can-i-push? install-commit-hook + uuid-check uuid-fix can-i-push? install-commit-hook gitflake8 diff --git a/scripts/check_duplicate_uuids.sh b/scripts/check_duplicate_uuids.sh deleted file mode 100755 index e8b81f9e0e2..00000000000 --- a/scripts/check_duplicate_uuids.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -# This script checks duplicated or empty uuids and exit with 1 if found - -# finds occurrences of empty @id: testimony tags -grep -E -i -r -n "@id:(.+[[:blank:]]|$)" tests/foreman/ -EMPTY=$? - -if [ $EMPTY = 0 ]; then - echo "Empty @id found in testimony tags" - exit 1 -fi - -# Finds occurrences of @id: in testimony tags then -# sort the output and filters only the duplicated -# then looks for existence of "@id:" in final output -# NOTE: can't print the line number -n here because of uniq -d -grep -r -i "@id:" tests/foreman/ | sort | uniq -d | grep "@id:" -DUPLICATE=$? - -# grep exits with status code 0 if text is found -# but we need to invert the logic here -# if duplicate found return with error 1 -if [ $DUPLICATE = 0 ]; then - echo "Duplicate @id found in testimony tags" - exit 1 -fi diff --git a/scripts/fix_uuids.sh b/scripts/fix_uuids.sh new file mode 100755 index 00000000000..fa45bbff0a9 --- /dev/null +++ b/scripts/fix_uuids.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash + +# This script finds empty @id: and replace with new uuids +# Then it finds for duplicates and replaces last occurrence with new uuids +# finally it fixes '@id:xyz' to '@id: xyz' (missing space after :) + +ID_TOKEN="@id:" + +if [ "$1" = "--check" ]; then + CHECK_ONLY=true + echo "Running in uuid-check only mode..." +else + CHECK_ONLY=false + echo "Running in uuid-fix mode..." +fi + +# finds occurrences of empty id: testimony tags +EMPTY_IDS=$(grep -E -i -r -n "${ID_TOKEN}(.+[[:blank:]]|$)" tests/foreman/ --include=*.py) + +if [ -n "$EMPTY_IDS" ]; then + if [ $CHECK_ONLY = true ]; then + echo "Empty $ID_TOKEN found in testimony tags" + echo $EMPTY_IDS + exit 1 + else + echo "Generating new UUIDS for empty $ID_TOKEN tags..." + fi +else + echo "No empty $ID_TOKEN was found" +fi + +# iterate if any empty @id found +for output_line in $EMPTY_IDS +do + if (echo "$output_line" | grep "tests/foreman"); then + OLDIFS=$IFS + # splits the grep output to get filename and occurrence line number + IFS=':' read -r filename line <<< $output_line + # generate uuid and place in specific line number + NEW_ID=$(python -c 'import uuid; print(uuid.uuid4())') + sed -r -i~ "${line}s/${ID_TOKEN}(.+[[:blank:]]|$)/${ID_TOKEN} ${NEW_ID}/g" $filename + IFS=$OLDIFS + fi +done + +# This script finds duplicated @id and replaces with new uuids + +# Finds occurrences of @id: in testimony tags then +# sort the output and filters only the duplicated +# then looks for existence of "@id:" in final output +# NOTE: can't print the line number -n here because of uniq -d +DUP_EXISTS=$(grep -r -i $ID_TOKEN tests/foreman/ --include=*.py | sort | uniq -d | grep $ID_TOKEN) + +if [ -n "$DUP_EXISTS" ]; then + if [ $CHECK_ONLY = true ]; then + echo "Duplicate $ID_TOKEN found in testimony tags" + echo $DUP_EXISTS + exit 1 + else + echo "Generating new UUIDS for duplicated $ID_TOKEN tags..." + fi +else + echo "No duplicated $ID_TOKEN was found" +fi + +grep -r -i $ID_TOKEN tests/foreman/ --include=*.py | sort | uniq -d | grep $ID_TOKEN | while read -r line ; do + OLDIFS=$IFS + IFS=':' read -r dup_file dup_id <<< $line + echo "filename: $dup_file" + echo "Id to replace: $dup_id" + NEW_ID=$(python -c 'import uuid; print(uuid.uuid4())') + echo "Replacing with the new id: $NEW_ID" + LAST_LINE=$(grep -i -n "$dup_id" $dup_file | tail -1) + + IFS=':' read -r linenumber linecontent <<< $LAST_LINE + echo $linenumber + trimmed_linecontent=$(echo $linecontent) + sed -i~ "${linenumber}s/${trimmed_linecontent}/${ID_TOKEN} ${NEW_ID}/g" $dup_file + echo "----------------------------------------------------------------" + IFS=$OLDIFS +done + +# This script finds id: missing spaces after : + + +MISSING_SPACES=$(grep -E -i -r -n "${ID_TOKEN}[^ ]" tests/foreman/ --include=*.py) + +if [ -n "$MISSING_SPACES" ]; then + if [ $CHECK_ONLY = true ]; then + echo "Found $ID_TOKEN tags missing space after : ..." + echo $MISSING_SPACES + exit 1 + else + echo "Fixing $ID_TOKEN tags missing spaces..." + fi +else + echo "No $ID_TOKEN missing spaces was found" +fi + +grep -E -i -r -n "${ID_TOKEN}[^ ]" tests/foreman/ --include=*.py | while read -r line ; do + OLDIFS=$IFS + IFS=':' read -r missing_file linenumber linecontent <<< $line + IFS=':' read -r tag uuid <<< $linecontent + trimmed_linecontent=$(echo $linecontent) + sed -i~ "${linenumber}s/${trimmed_linecontent}/${tag}: ${uuid}/g" $missing_file + IFS=$OLDIFS +done diff --git a/scripts/replace_dup_uuids.sh b/scripts/replace_dup_uuids.sh deleted file mode 100755 index df5bb0109fb..00000000000 --- a/scripts/replace_dup_uuids.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script finds duplicated @id and replaces with new uuids - -grep -r -i "@id:" tests/foreman/ | sort | uniq -d | grep "@id:" | while read -r line ; do - OLDIFS=$IFS - IFS=':' read -r dup_file dup_id <<< $line - echo "filename: $dup_file" - echo "Id to replace: $dup_id" - NEW_ID=$(uuidgen) - echo "Replacing with the new id: $NEW_ID" - LAST_LINE=$(grep -i -n "$dup_id" $dup_file | tail -1) - - IFS=':' read -r linenumber linecontent <<< $LAST_LINE - echo $linenumber - trimmed_linecontent=$(echo $linecontent) - sed -i~ "${linenumber}s/${trimmed_linecontent}/@id: ${NEW_ID}/g" $dup_file - echo "----------------------------------------------------------------" - IFS=$OLDIFS -done diff --git a/scripts/replace_empty_uuids.sh b/scripts/replace_empty_uuids.sh deleted file mode 100755 index 9dc96db0c0a..00000000000 --- a/scripts/replace_empty_uuids.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash - -# this script finds empty @id: and replace with new uuids - -# finds occurrences of empty @id: testimony tags -EMPTY_IDS=$(grep -E -i -r -n "@id:(.+[[:blank:]]|$)" tests/foreman/) - -if [ -n "$EMPTY_IDS" ]; then - echo "Generating new UUIDS for empty @id tags..." -else - echo "No empty @id was found" -fi - -# iterate if any empty @id found -for output_line in $EMPTY_IDS -do - if (echo "$output_line" | grep "tests/foreman"); then - OLDIFS=$IFS - # splits the grep output to get filename and occurrence line number - IFS=':' read -r filename line <<< $output_line - # generate uuid and place in specific line number - sed -r -i~ "${line}s/@id:(.+[[:blank:]]|$)/@id: $(uuidgen)/g" $filename - IFS=$OLDIFS - fi -done