Merge branch '88-blockscraper-implementation' into asdt

Author: VinciGit00

.github/workflows/pylint.yml

@@ -1,5 +1,3 @@
-name: Pylint
-
 on: [push]
 
 jobs:
@@ -20,4 +18,13 @@ jobs:
           pip install pylint
           pip install -r requirements.txt
       - name: Analysing the code with pylint
-        run: pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py scrapegraphai/*.py 
+        run: pylint --disable=C0114,C0115,C0116 --exit-zero scrapegraphai/**/*.py scrapegraphai/*.py
+      - name: Check Pylint score
+        run: |
+          pylint_score=$(pylint --disable=all --enable=metrics --output-format=text scrapegraphai/**/*.py scrapegraphai/*.py | grep 'Raw metrics' | awk '{print $4}')
+          if (( $(echo "$pylint_score < 8" | bc -l) )); then
+            echo "Pylint score is below 8. Blocking commit."
+            exit 1
+          else
+            echo "Pylint score is acceptable."
+          fi

.github/workflows/release.yml

@@ -0,0 +1,79 @@
+name: Release
+on:
+  push:
+    branches:
+      - main
+      - pre/*
+
+jobs:
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    steps:
+      - name: Install git
+        run: |
+          sudo apt update
+          sudo apt install -y git
+      - name: Install Python Env and Poetry
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+      - run: pip install poetry
+      - name: Install Node Env
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Checkout
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Build app
+        run: |
+          poetry install
+          poetry build
+        id: build_cache
+        if: success()
+      - name: Cache build
+        uses: actions/cache@v2
+        with:
+          path: ./dist
+          key: ${{ runner.os }}-build-${{ hashFiles('dist/**') }}
+        if: steps.build_cache.outputs.id != ''
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: build
+    environment: development
+    if: | 
+        github.event_name == 'push' && github.ref == 'refs/heads/main' || 
+        github.event_name == 'push' && github.ref == 'refs/heads/pre/beta' || 
+        github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged && github.event.pull_request.base.ref == 'main' || 
+        github.event_name == 'pull_request' && github.event.action == 'closed' && github.event.pull_request.merged && github.event.pull_request.base.ref == 'pre/beta'
+    permissions:
+      contents: write
+      issues: write
+      pull-requests: write
+      id-token: write
+    steps:
+      - name: Checkout repo
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - name: Semantic Release
+        uses: cycjimmy/[email protected]
+        with:
+          semantic_version: 23
+          extra_plugins: |
+            semantic-release-pypi@3
+            @semantic-release/git 
+            @semantic-release/commit-analyzer@12 
+            @semantic-release/release-notes-generator@13 
+            @semantic-release/github@10 
+            @semantic-release/changelog@6
+            conventional-changelog-conventionalcommits@7
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}

.gitignore

@@ -28,6 +28,11 @@ venv/
 *.sqlite
 *.google-cookie
 examples/graph_examples/ScrapeGraphAI_generated_graph
-examples/**/*.csv
-examples/**/*.json
+examples/**/result.csv
+examples/**/result.json
 main.py
+poetry.lock
+
+# lock files
+*.lock
+poetry.lock

.releaserc.yml

@@ -0,0 +1,56 @@
+plugins:
+  - - "@semantic-release/commit-analyzer"
+    - preset: conventionalcommits
+  - - "@semantic-release/release-notes-generator"
+    - writerOpts:
+        commitsSort:
+        - subject
+        - scope
+      preset: conventionalcommits
+      presetConfig:
+        types:
+        - type: feat
+          section: Features
+        - type: fix
+          section: Bug Fixes
+        - type: chore
+          section: chore
+        - type: docs
+          section: Docs
+        - type: style
+          hidden: true
+        - type: refactor
+          section: Refactor
+        - type: perf
+          section: Perf
+        - type: test
+          section: Test
+        - type: build
+          section: Build
+        - type: ci
+          section: CI
+  - "@semantic-release/changelog"
+  - "semantic-release-pypi"
+  - "@semantic-release/github"
+  - - "@semantic-release/git"
+    - assets:
+        - CHANGELOG.md
+        - pyproject.toml
+      message: |-
+        ci(release): ${nextRelease.version} [skip ci]
+
+        ${nextRelease.notes}
+branches:
+  #child branches coming from tagged version for bugfix (1.1.x) or new features (1.x)
+  #maintenance branch
+  - name: "+([0-9])?(.{+([0-9]),x}).x"
+    channel: "stable"
+  #release a production version when merging towards main
+  - name: "main"
+    channel: "stable"
+  #prerelease branch
+  - name: "pre/beta"
+    channel: "dev"
+    prerelease: "beta"
+debug: true
+