[UPDATE] Add pre-commit hooks to catch linting issues

remediation/precommit/precommit-config.yml

@@ -0,0 +1,61 @@
+hooks:
+  common:
+    - repo: https://github.com/pre-commit/pre-commit-hooks
+      rev: v4.4.0
+      hooks:
+        - id: end-of-file-fixer
+        - id: trailing-whitespace
+    - repo: https://github.com/gitleaks/gitleaks
+      rev: v8.16.3
+      hooks:
+        - id: gitleaks
+  Python: 
+    - repo: https://github.com/pylint-dev/pylint
+      rev: v2.17.2
+      hooks:
+        - id: pylint
+  JavaScript: 
+    - repo: https://github.com/pre-commit/mirrors-eslint
+      rev: v8.38.0
+      hooks:
+        - id: eslint
+  TypeScript: 
+    - repo: https://github.com/pre-commit/mirrors-eslint
+      rev: v8.38.0
+      hooks:
+        - id: eslint
+  Java: 
+    - repo: https://github.com/gherynos/pre-commit-java
+      rev: v0.2.4
+      hooks:
+        - id: Checkstyle
+  C: 
+    - repo: https://github.com/pocc/pre-commit-hooks
+      rev: v1.3.5
+      hooks:
+        - id: cpplint
+  C++: 
+    - repo: https://github.com/pocc/pre-commit-hooks
+      rev: v1.3.5
+      hooks:
+        - id: cpplint
+  PHP: 
+    - repo: https://github.com/digitalpulp/pre-commit-php
+      rev: 1.4.0
+      hooks:
+        - id: php-lint-all
+  Ruby: 
+    - repo: https://github.com/jumanjihouse/pre-commit-hooks
+      rev: 3.0.0
+      hooks:
+        - id: RuboCop
+  Go: 
+    - repo: https://github.com/golangci/golangci-lint
+      rev: v1.52.2
+      hooks:
+        - id: golangci-lint
+  Shell: 
+    - repo: https://github.com/jumanjihouse/pre-commit-hooks
+      rev: 3.0.0
+      hooks:
+        - id: shellcheck

remediation/precommit/precommitconfig.go

@@ -0,0 +1,212 @@
+package precommit
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"sort"
+	"strings"
+
+	"github.com/step-security/secure-repo/remediation/workflow/permissions"
+	"gopkg.in/yaml.v3"
+)
+
+type UpdatePrecommitConfigResponse struct {
+	OriginalInput        string
+	FinalOutput          string
+	IsChanged            bool
+	ConfigfileFetchError bool
+}
+
+type UpdatePrecommitConfigRequest struct {
+	Content   string
+	Languages []string
+}
+
+type PrecommitConfig struct {
+	Repos []Repo `yaml:"repos"`
+}
+
+type Repo struct {
+	Repo  string `yaml:"repo"`
+	Rev   string `yaml:"rev"`
+	Hooks []Hook `yaml:"hooks"`
+}
+
+type Hook struct {
+	Id string `yaml:"id"`
+}
+
+type FetchPrecommitConfig struct {
+	Hooks Hooks `yaml:"hooks"`
+}
+
+// type LangHook struct {
+// 	Repo Repo `yaml:"hook"`
+// }
+
+type Hooks map[string][]Repo
+
+func getConfigFile() (string, error) {
+	filePath := os.Getenv("PRECOMMIT_CONFIG")
+
+	if filePath == "" {
+		filePath = "./"
+	}
+
+	configFile, err := ioutil.ReadFile(path.Join(filePath, "precommit-config.yml"))
+	if err != nil {
+		return "", err
+	}
+
+	return string(configFile), nil
+}
+
+func GetHooks(languages []string, alreadyPresentHooks map[string]bool) ([]Repo, error) {
+	configFile, err := getConfigFile()
+	if err != nil {
+		return nil, err
+	}
+	var fetchPrecommitConfig FetchPrecommitConfig
+	yaml.Unmarshal([]byte(configFile), &fetchPrecommitConfig)
+	newHooks := make(map[string]Repo)
+	for _, lang := range languages {
+		if _, ok := alreadyPresentHooks[fetchPrecommitConfig.Hooks[lang][0].Hooks[0].Id]; !ok {
+			if repo, ok := newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo]; ok {
+				repo.Hooks = append(repo.Hooks, fetchPrecommitConfig.Hooks[lang][0].Hooks...)
+				newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo] = repo
+			} else {
+				newHooks[fetchPrecommitConfig.Hooks[lang][0].Repo] = fetchPrecommitConfig.Hooks[lang][0]
+			}
+			alreadyPresentHooks[fetchPrecommitConfig.Hooks[lang][0].Hooks[0].Id] = true
+		}
+	}
+	// Adding common hooks
+	var repos []Repo
+	for _, repo := range fetchPrecommitConfig.Hooks["common"] {
+		tempRepo := repo
+		tempRepo.Hooks = nil
+		hookPresent := false
+		for _, hook := range repo.Hooks {
+			if _, ok := alreadyPresentHooks[hook.Id]; !ok {
+				tempRepo.Hooks = append(tempRepo.Hooks, hook)
+				hookPresent = true
+			}
+		}
+		if hookPresent {
+			repos = append(repos, tempRepo)
+		}
+	}
+	for _, repo := range newHooks {
+		repos = append(repos, repo)
+	}
+	sort.Slice(repos, func(i, j int) bool {
+		return repos[i].Repo < repos[j].Repo
+	})
+	return repos, nil
+}
+
+func UpdatePrecommitConfig(precommitConfig string) (*UpdatePrecommitConfigResponse, error) {
+	var updatePrecommitConfigRequest UpdatePrecommitConfigRequest
+	json.Unmarshal([]byte(precommitConfig), &updatePrecommitConfigRequest)
+	inputConfigFile := []byte(updatePrecommitConfigRequest.Content)
+	configMetadata := PrecommitConfig{}
+	err := yaml.Unmarshal(inputConfigFile, &configMetadata)
+	if err != nil {
+		return nil, err
+	}
+
+	response := new(UpdatePrecommitConfigResponse)
+	response.FinalOutput = updatePrecommitConfigRequest.Content
+	response.OriginalInput = updatePrecommitConfigRequest.Content
+	response.IsChanged = false
+
+	if updatePrecommitConfigRequest.Content == "" {
+		response.FinalOutput = "repos:"
+	}
+
+	alreadyPresentHooks := make(map[string]bool)
+	for _, repos := range configMetadata.Repos {
+		for _, hook := range repos.Hooks {
+			alreadyPresentHooks[hook.Id] = true
+		}
+	}
+	// Contains a list of hooks to be added and not present in the file
+	Hooks, err := GetHooks(updatePrecommitConfigRequest.Languages, alreadyPresentHooks)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, Update := range Hooks {
+		repoAlreadyExist := false
+		for _, update := range configMetadata.Repos {
+			if update.Repo == Update.Repo {
+				repoAlreadyExist = true
+			}
+			if repoAlreadyExist {
+				break
+			}
+		}
+		response.FinalOutput, err = addHook(Update, repoAlreadyExist, response.FinalOutput)
+		if err != nil {
+			return nil, err
+		}
+		response.IsChanged = true
+	}
+
+	return response, nil
+}
+
+func addHook(Update Repo, repoAlreadyExist bool, inputYaml string) (string, error) {
+	t := yaml.Node{}
+
+	err := yaml.Unmarshal([]byte(inputYaml), &t)
+	if err != nil {
+		return "", fmt.Errorf("unable to parse yaml %v", err)
+	}
+
+	spaces := ""
+	jobNode := permissions.IterateNode(&t, "hooks", "!!seq", 0)
+	if jobNode == nil {
+		spaces = "  "
+	} else {
+		for i := 0; i < jobNode.Column-1; i++ {
+			spaces += " "
+		}
+	}
+
+	if repoAlreadyExist {
+		jobNode = permissions.IterateNode(&t, Update.Repo, "!!str", 0)
+		if jobNode == nil {
+			return "", fmt.Errorf("Repo Name %s not found in the input yaml", Update.Repo)
+		}
+
+		// TODO: Also update rev version for already exist repo
+		inputLines := strings.Split(inputYaml, "\n")
+		var output []string
+		for i := 0; i < jobNode.Line+1; i++ {
+			output = append(output, inputLines[i])
+		}
+
+		for _, hook := range Update.Hooks {
+			output = append(output, spaces+fmt.Sprintf("- id: %s", hook.Id))
+		}
+
+		for i := jobNode.Line + 1; i < len(inputLines); i++ {
+			output = append(output, inputLines[i])
+		}
+		return strings.Join(output, "\n"), nil
+	} else {
+		inputLines := strings.Split(inputYaml, "\n")
+		inputLines = append(inputLines, fmt.Sprintf("- repo: %s", Update.Repo))
+		inputLines = append(inputLines, fmt.Sprintf("  rev: %s", Update.Rev))
+		inputLines = append(inputLines, fmt.Sprintf("  hooks:"))
+
+		for _, hook := range Update.Hooks {
+			inputLines = append(inputLines, spaces+fmt.Sprintf("- id: %s", hook.Id))
+		}
+		return strings.Join(inputLines, "\n"), nil
+	}
+}

remediation/precommit/precommitconfig_test.go

@@ -0,0 +1,72 @@
+package precommit
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"path"
+	"testing"
+)
+
+func TestUpdatePrecommitConfig(t *testing.T) {
+
+	const inputDirectory = "../../testfiles/precommit/input"
+	const outputDirectory = "../../testfiles/precommit/output"
+
+	tests := []struct {
+		fileName  string
+		Languages []string
+		isChanged bool
+	}{
+		{
+			fileName:  "basic.yml",
+			Languages: []string{"JavaScript", "C++"},
+			isChanged: true,
+		},
+		{
+			fileName:  "file-not-exit.yml",
+			Languages: []string{"JavaScript", "C++"},
+			isChanged: true,
+		},
+		{
+			fileName:  "same-repo-different-hooks.yml",
+			Languages: []string{"Ruby", "Shell"},
+			isChanged: true,
+		},
+	}
+
+	for _, test := range tests {
+		var updatePrecommitConfigRequest UpdatePrecommitConfigRequest
+		input, err := ioutil.ReadFile(path.Join(inputDirectory, test.fileName))
+		if err != nil {
+			log.Fatal(err)
+		}
+		updatePrecommitConfigRequest.Content = string(input)
+		updatePrecommitConfigRequest.Languages = test.Languages
+		inputRequest, err := json.Marshal(updatePrecommitConfigRequest)
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		output, err := UpdatePrecommitConfig(string(inputRequest))
+		if err != nil {
+			t.Fatalf("Error not expected: %s", err)
+		}
+
+		expectedOutput, err := ioutil.ReadFile(path.Join(outputDirectory, test.fileName))
+		if err != nil {
+			log.Fatal(err)
+		}
+
+		if string(expectedOutput) != output.FinalOutput {
+			t.Errorf("test failed %s did not match expected output\n%s", test.fileName, output.FinalOutput)
+		}
+
+		if output.IsChanged != test.isChanged {
+			t.Errorf("test failed %s did not match IsChanged, Expected: %v Got: %v", test.fileName, test.isChanged, output.IsChanged)
+
+		}
+
+	}
+
+}

testfiles/precommit/input/basic.yml

@@ -0,0 +1,19 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+    - id: check-yaml
+    - id: check-json
+    - id: trailing-whitespace
+- repo: https://github.com/pre-commit/mirrors-eslint
+  rev: v8.23.0
+  hooks:
+    - id: eslint
+- repo: https://github.com/ejba/pre-commit-maven
+  rev: v0.3.3
+  hooks:
+    - id: maven-test
+- repo: https://github.com/zricethezav/gitleaks
+  rev: v8.12.0
+  hooks:
+    - id: gitleaks

testfiles/precommit/input/same-repo-different-hooks.yml

@@ -0,0 +1,19 @@
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v2.3.0
+  hooks:
+    - id: check-yaml
+    - id: check-json
+    - id: trailing-whitespace
+- repo: https://github.com/pre-commit/mirrors-eslint
+  rev: v8.23.0
+  hooks:
+    - id: eslint
+- repo: https://github.com/ejba/pre-commit-maven
+  rev: v0.3.3
+  hooks:
+    - id: maven-test
+- repo: https://github.com/zricethezav/gitleaks
+  rev: v8.12.0
+  hooks:
+    - id: gitleaks